Initial import: shopdb-flask multi-site asset platform

Flask + Vue rewrite of the classic-ASP shopdb, built as a plugin framework so sister GE Aerospace sites can adopt it. Product version 0.7.0-pre-0.8.0, plugin contract 0.11.0, 966 backend tests. Key entry points: - docs/INSTALL-WINDOWS-IIS.md - tested Windows + IIS install runbook (own-site and subpath deployment methods) - docs/PILOT-DEPLOY.md - production pilot: stand up + import + verify - docs/IMPORT-API.md - schema-agnostic legacy data import contract - scripts/site_imports/wjf/ - West Jefferson reference import loader - docs/adr/ - twelve accepted architecture decision records
2026-07-14 07:58:51 -04:00
commit a9d0dfff4a
599 changed files with 88557 additions and 0 deletions
--- a/scripts/check-naming-and-style.sh
+++ b/scripts/check-naming-and-style.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+#
+# Pre-commit naming + style check for shopdb-flask.
+# Enforces CONTRIBUTING.md rules:
+#   1. No non-ASCII chars in source (em-dashes, smart quotes, arrows, emojis)
+#   2. No banned shorthand identifiers (cfg, ctx, mgr, req, res, env, util, helper)
+#      as standalone names (suffix usage like printers_bp, request_obj is allowed)
+#   3. No snake_case DB column names in __tablename__ or db.Column attrs
+#   4. No snake_case API params in frontend that should match DB column names
+#
+# Exits non-zero if any violation found.
+# Skips: venv/, node_modules/, __pycache__/, frontend/dist/, migrations/versions/
+
+set -e
+
+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)"
+cd "$REPO_ROOT"
+
+VIOLATIONS=0
+
+EXCLUDES=(
+    --exclude-dir=venv
+    --exclude-dir=node_modules
+    --exclude-dir=__pycache__
+    --exclude-dir=dist
+    --exclude-dir=.git
+    --exclude-dir=versions
+)
+
+INCLUDES_CODE=(
+    --include='*.py'
+    --include='*.vue'
+    --include='*.js'
+    --include='*.ts'
+)
+
+INCLUDES_ALL=(
+    --include='*.py'
+    --include='*.vue'
+    --include='*.js'
+    --include='*.ts'
+    --include='*.json'
+    --include='*.md'
+    --include='*.yaml'
+    --include='*.yml'
+)
+
+echo "==> Checking for non-ASCII characters..."
+NON_ASCII=$(grep -rPn '[^\x00-\x7F]' "${EXCLUDES[@]}" "${INCLUDES_CODE[@]}" . 2>/dev/null || true)
+if [ -n "$NON_ASCII" ]; then
+    echo "FAIL: non-ASCII characters found (em-dashes, smart quotes, arrows, emojis):"
+    echo "$NON_ASCII"
+    echo
+    VIOLATIONS=$((VIOLATIONS + 1))
+fi
+
+echo "==> Checking for banned shorthand (standalone)..."
+# Match the word as a standalone identifier: not preceded or followed by underscore/word char
+# Word boundary in grep is \b but we want to exclude suffix usage like printers_bp
+# So: match (^|[^a-zA-Z0-9_])(banned)([^a-zA-Z0-9_]|$)
+for word in cfg ctx mgr req res; do
+    HITS=$(grep -rPn "(^|[^a-zA-Z0-9_])${word}([^a-zA-Z0-9_]|\$)" "${EXCLUDES[@]}" --include='*.py' --include='*.vue' --include='*.js' --include='*.ts' . 2>/dev/null \
+        | grep -vP "(^|[^a-zA-Z0-9_])(request_obj|response_obj)" \
+        || true)
+    if [ -n "$HITS" ]; then
+        echo "FAIL: banned shorthand '$word' (standalone) found:"
+        echo "$HITS"
+        echo
+        VIOLATIONS=$((VIOLATIONS + 1))
+    fi
+done
+
+echo "==> Checking for snake_case DB tablenames..."
+SNAKE_TABLES=$(grep -rPn "__tablename__\s*=\s*['\"][^'\"]*_" "${EXCLUDES[@]}" --include='*.py' . 2>/dev/null || true)
+if [ -n "$SNAKE_TABLES" ]; then
+    echo "FAIL: snake_case __tablename__ found (must be lowercase concatenated):"
+    echo "$SNAKE_TABLES"
+    echo
+    VIOLATIONS=$((VIOLATIONS + 1))
+fi
+
+echo "==> Checking for snake_case DB column attrs..."
+SNAKE_COLS=$(grep -rPn "^\s+[a-z]+_[a-z_]+\s*=\s*db\.Column" "${EXCLUDES[@]}" --include='*.py' . 2>/dev/null || true)
+if [ -n "$SNAKE_COLS" ]; then
+    echo "FAIL: snake_case db.Column attribute found (must match column name, no underscores):"
+    echo "$SNAKE_COLS"
+    echo
+    VIOLATIONS=$((VIOLATIONS + 1))
+fi
+
+echo "==> Checking for snake_case ForeignKey targets..."
+SNAKE_FK=$(grep -rPn "ForeignKey\(['\"][^'\"]*_[^'\"]*['\"]" "${EXCLUDES[@]}" --include='*.py' . 2>/dev/null || true)
+if [ -n "$SNAKE_FK" ]; then
+    echo "FAIL: snake_case ForeignKey target found:"
+    echo "$SNAKE_FK"
+    echo
+    VIOLATIONS=$((VIOLATIONS + 1))
+fi
+
+echo "==> Checking for snake_case API params in frontend (DB-mirrored fields)..."
+SNAKE_FE=$(grep -rPn "params\.(machine_id|location_id|vendor_id|type_id|business_unit_id|model_id|status_id|operating_system_id|asset_id|user_id|is_active|is_shopfloor)" "${EXCLUDES[@]}" --include='*.vue' --include='*.js' --include='*.ts' . 2>/dev/null || true)
+if [ -n "$SNAKE_FE" ]; then
+    echo "FAIL: snake_case API params in frontend (must match DB column names without underscores):"
+    echo "$SNAKE_FE"
+    echo
+    VIOLATIONS=$((VIOLATIONS + 1))
+fi
+
+if [ "$VIOLATIONS" -gt 0 ]; then
+    echo "=================================================="
+    echo "$VIOLATIONS naming/style violation(s) found."
+    echo "See CONTRIBUTING.md for the full convention."
+    echo "=================================================="
+    exit 1
+fi
+
+echo "==> All naming/style checks passed."
+exit 0
--- a/scripts/migration/README.md
+++ b/scripts/migration/README.md
@@ -0,0 +1,24 @@
+# scripts/migration (superseded)
+
+The direct-SQL migrators that used to live here (`migrate_assets.py`,
+`migrate_communications.py`, `migrate_notifications.py`, `migrate_usb.py`,
+`run_migration.py`, `verify_migration.py`) and `scripts/import_from_mysql.py`
+were **removed** - they had drifted badly out of sync with the current schema
+(they targeted a nonexistent `equipment` table, the retired `Machine` model, and
+columns that no longer exist) and were actively misleading.
+
+## Use the import API + a site loader instead
+
+Legacy data is imported through the maintained, schema-agnostic contract in
+the IMPORT-API page in the project wiki (create through real endpoints
+with an admin PAT + `X-Import-Mode`). Each adopting site writes a thin loader
+that reads its own source database and drives that API.
+
+- Reference implementation: [`scripts/site_imports/wjf/`](../site_imports/wjf/)
+  (West Jefferson classic-ASP -> flask).
+- Adoption playbook: the IMPORT-ADOPTION page in the project wiki.
+
+## What's kept here
+
+- `fix_legacy_schema.sql` - one-time SQL fixups against a legacy source DB.
+- `one-offs/` - individual one-shot SQL snippets (see its README).
--- a/scripts/migration/init.py
+++ b/scripts/migration/init.py
@@ -0,0 +1 @@
+"""Data migration scripts for VBScript ShopDB to Flask migration."""
--- a/scripts/migration/fix_legacy_schema.sql
+++ b/scripts/migration/fix_legacy_schema.sql
@@ -0,0 +1,88 @@
+-- =============================================================================
+-- Legacy Schema Migration Script
+-- Adds missing columns to make VBScript ShopDB compatible with Flask ShopDB
+-- =============================================================================
+
+-- -----------------------------------------------------------------------------
+-- appowners table
+-- -----------------------------------------------------------------------------
+ALTER TABLE appowners
+  ADD COLUMN IF NOT EXISTS email VARCHAR(100) NULL,
+  ADD COLUMN IF NOT EXISTS phone VARCHAR(50) NULL,
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS isactive TINYINT(1) DEFAULT 1;
+
+-- -----------------------------------------------------------------------------
+-- pctypes / computertypes alignment
+-- -----------------------------------------------------------------------------
+-- The Flask app uses pctype table but expects certain columns
+ALTER TABLE pctype
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW();
+
+-- Ensure isactive is correct type
+ALTER TABLE pctype MODIFY COLUMN isactive TINYINT(1) DEFAULT 1;
+
+-- -----------------------------------------------------------------------------
+-- statuses table (if needed - Flask uses assetstatuses)
+-- -----------------------------------------------------------------------------
+-- assetstatuses already populated earlier
+
+-- -----------------------------------------------------------------------------
+-- subnets table
+-- -----------------------------------------------------------------------------
+ALTER TABLE subnets
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS isactive TINYINT(1) DEFAULT 1;
+
+-- -----------------------------------------------------------------------------
+-- vlans table
+-- -----------------------------------------------------------------------------
+ALTER TABLE vlans
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS isactive TINYINT(1) DEFAULT 1;
+
+-- -----------------------------------------------------------------------------
+-- usbdevices table
+-- -----------------------------------------------------------------------------
+ALTER TABLE usbdevices
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS isactive TINYINT(1) DEFAULT 1;
+
+-- -----------------------------------------------------------------------------
+-- usbcheckouts table
+-- -----------------------------------------------------------------------------
+ALTER TABLE usbcheckouts
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW();
+
+-- -----------------------------------------------------------------------------
+-- notifications table
+-- -----------------------------------------------------------------------------
+ALTER TABLE notifications
+  ADD COLUMN IF NOT EXISTS createddate DATETIME DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS modifieddate DATETIME DEFAULT NOW();
+
+-- Copy dates from existing columns if they exist
+UPDATE notifications SET createddate = startdate WHERE createddate IS NULL;
+UPDATE notifications SET modifieddate = startdate WHERE modifieddate IS NULL;
+
+-- -----------------------------------------------------------------------------
+-- Verify key counts
+-- -----------------------------------------------------------------------------
+SELECT 'Migration complete. Record counts:' as status;
+SELECT 'vendors' as tbl, COUNT(*) as cnt FROM vendors
+UNION ALL SELECT 'models', COUNT(*) FROM models
+UNION ALL SELECT 'machinetypes', COUNT(*) FROM machinetypes
+UNION ALL SELECT 'operatingsystems', COUNT(*) FROM operatingsystems
+UNION ALL SELECT 'businessunits', COUNT(*) FROM businessunits
+UNION ALL SELECT 'applications', COUNT(*) FROM applications
+UNION ALL SELECT 'machines', COUNT(*) FROM machines
+UNION ALL SELECT 'printers', COUNT(*) FROM printers
+UNION ALL SELECT 'assets', COUNT(*) FROM assets
+UNION ALL SELECT 'knowledgebase', COUNT(*) FROM knowledgebase
+UNION ALL SELECT 'notifications', COUNT(*) FROM notifications;
--- a/scripts/migration/one-offs/README.md
+++ b/scripts/migration/one-offs/README.md
@@ -0,0 +1,16 @@
+# One-off migration scripts (historical)
+
+These are hand-run SQL scripts kept for reference. They are NOT part of the
+Alembic chain (`migrations/versions/`) and are not run by `flask db upgrade`. A
+fresh install never needs them; they exist for databases that predate the
+corresponding change.
+
+- `add_recertification_type.sql` - inserts the blue "Recertification"
+  notification type into an already-existing database. Historical: the
+  notifications plugin now seeds this type on install
+  (`plugins/notifications/plugin.py`), so any fresh install already has it. Only
+  a pre-existing DB that was never re-installed needs this script. Idempotent.
+
+A related one-off, widening `notifications.employeesso` / `employeename` to
+TEXT, has been removed because it is fully superseded by Alembic migration
+`7d02_widen_notification_employee_cols`.
--- a/scripts/migration/one-offs/add_recertification_type.sql
+++ b/scripts/migration/one-offs/add_recertification_type.sql
@@ -0,0 +1,21 @@
+-- Recertification notification type (shopdb-flask)
+--
+-- Employee-photo type (blue, like recognition) for employees who must retake a
+-- training course. The shopfloor feed splits it one card per employee and the
+-- dashboard shows it in a blue "Recertification Required" carousel; the display
+-- window is two weeks (computed server-side in the notifications API).
+--
+-- Idempotent - safe to re-run. The notifications plugin also seeds this type on
+-- install (plugins/notifications/plugin.py); this file is for databases that
+-- already exist and are not re-installed.
+
+INSERT INTO notificationtypes (typename, typedescription, typecolor, isactive)
+SELECT * FROM (
+    SELECT 'Recertification' AS typename,
+           'Employees due to retake a training course (shows two weeks)' AS typedescription,
+           'recertification' AS typecolor,
+           1 AS isactive
+) AS t
+WHERE NOT EXISTS (
+    SELECT 1 FROM notificationtypes WHERE typename = 'Recertification'
+);
--- a/scripts/site_imports/init.py
+++ b/scripts/site_imports/init.py
--- a/scripts/site_imports/wjf/README.md
+++ b/scripts/site_imports/wjf/README.md
@@ -0,0 +1,48 @@
+# WJ classic-ASP import loader (reference implementation)
+
+This is **site-specific reference glue**, not product code. It maps West
+Jefferson's classic-ASP `shopdb` / `cmmc_usb` / `wjf_employees` schema onto the
+maintained, schema-agnostic import contract in the IMPORT-API wiki page.
+
+**Every adopting site has its own source database.** Nobody else runs this
+loader. Instead, copy the pattern:
+
+1. Point the harness at your source DB(s) (edit `harness.Source`).
+2. Write per-entity stages that read your tables and POST to the same
+   the IMPORT-API wiki page endpoints with `Authorization: Bearer <admin PAT>` and
+   `X-Import-Mode: true`.
+3. Persist legacy-id -> new-id crosswalks (see `harness.IdMap`) so later stages
+   resolve foreign keys and a crashed run resumes.
+
+The import API is the stable contract; loaders are per-site. The mapping itself
+is the bulk of the work: enumerate the source tables against this contract and
+write one stage per entity (this loader is the worked example).
+
+## Running (against a THROWAWAY import database)
+
+```bash
+DATABASE_URL='mysql+pymysql://root:PW@127.0.0.1:3306/shopdb_flask_import?charset=utf8mb4' \
+  venv/bin/python -m scripts.site_imports.wjf.run --stages reference,employees
+```
+
+Prereqs: a fresh target DB built with `flask db upgrade` + `flask plugin
+upgrade-all` + `flask seed permissions/settings/reference-data`, and the three
+source dumps loaded into scratch DBs (`shopdb_src`, `cmmc_usb_src`,
+`wjf_employees_src`). See `scratchpad/IMPORT-PLAN.md` for the full mapping,
+resolved decisions, and remaining stages.
+
+## Status
+
+Complete - all 15 stages built and verified end-to-end against a fresh scratch
+target (zero endpoint errors): `reference`, `employees`, `catalog`, `assets`
+(the hub + machineid->assetid crosswalk), `locations`, `printers`,
+`communications`, `applications` (+ installs), `warranties`, `notifications`,
+`knowledgebase`, `relationships`, `subnets`, `usb`, `verify`.
+
+Last full run: 983 assets (computer 663, machine 76, network 58, measuring-tool
+136, printer 50), 24 locations, 415 employees, 850 installs, 461 primary IPs,
+464 warranties, 261 notifications, 341 KB, 93 relationships, 37 subnets, 18 USB
+devices + 232 events. UI spot-check passed (computer list + shop-floor map).
+
+Known follow-ups: general asset `locationid` is null (no source column outside
+printers); the ~146 duplicate machinenumbers are first-wins-skipped by decision.
--- a/scripts/site_imports/wjf/init.py
+++ b/scripts/site_imports/wjf/init.py
--- a/scripts/site_imports/wjf/harness.py
+++ b/scripts/site_imports/wjf/harness.py
@@ -0,0 +1,177 @@
+"""Import harness for the WJ classic-ASP -> flask reference loader.
+
+This is SITE-SPECIFIC reference glue, NOT product code. It maps West Jefferson's
+classic schema onto the maintained, schema-agnostic import contract in
+the IMPORT-API wiki page. Other sites copy this pattern against their own source DB;
+they do not run this loader.
+
+The harness builds the flask app against the current DATABASE_URL (point it at a
+throwaway import database), mints an unscoped admin PAT in-process, and drives
+the REAL import endpoints through the app test client with the X-Import-Mode
+header - so it exercises the same routes/authz/validation an HTTP client would,
+without needing a running server. Legacy-id -> new-id maps persist to JSON so a
+crashed run resumes and later stages resolve foreign keys.
+"""
+
+import json
+import os
+import pymysql
+
+from shopdb import create_app
+from shopdb.extensions import db
+
+
+def _source_password():
+    """Root password for the scratch source DBs: reuse the dev DATABASE_URL in
+    .env unless SOURCE_DB_PASSWORD overrides."""
+    if os.environ.get('SOURCE_DB_PASSWORD'):
+        return os.environ['SOURCE_DB_PASSWORD']
+    env = os.path.join(os.path.dirname(__file__), '..', '..', '..', '.env')
+    with open(os.path.abspath(env)) as handle:
+        for line in handle:
+            if line.startswith('DATABASE_URL='):
+                after = line.split('root:', 1)[1]
+                return after.split('@', 1)[0]
+    raise RuntimeError('could not resolve source DB password')
+
+
+class Source:
+    """Read-only access to the three scratch source databases."""
+
+    def __init__(self, host='127.0.0.1', port=3306, user='root', password=None):
+        self.password = password or _source_password()
+        self.host, self.port, self.user = host, port, user
+        self._conns = {}
+
+    def db(self, name):
+        if name not in self._conns:
+            self._conns[name] = pymysql.connect(
+                host=self.host, port=self.port, user=self.user,
+                password=self.password, database=name, charset='utf8mb4',
+                cursorclass=pymysql.cursors.DictCursor)
+        return self._conns[name]
+
+    def rows(self, dbname, sql, args=None):
+        with self.db(dbname).cursor() as cur:
+            cur.execute(sql, args or ())
+            return cur.fetchall()
+
+
+class IdMap:
+    """Persisted legacy-id -> new-id crosswalks, one namespace per entity."""
+
+    def __init__(self, path):
+        self.path = path
+        self._data = {}
+        if os.path.exists(path):
+            with open(path) as handle:
+                self._data = json.load(handle)
+
+    def put(self, namespace, legacy, new):
+        self._data.setdefault(namespace, {})[str(legacy)] = new
+        self.save()
+
+    def get(self, namespace, legacy, default=None):
+        return self._data.get(namespace, {}).get(str(legacy), default)
+
+    def has(self, namespace, legacy):
+        return str(legacy) in self._data.get(namespace, {})
+
+    def count(self, namespace):
+        return len(self._data.get(namespace, {}))
+
+    def save(self):
+        with open(self.path, 'w') as handle:
+            json.dump(self._data, handle, indent=1)
+
+
+class Harness:
+    """App + admin PAT + import-mode test client + id maps."""
+
+    def __init__(self, idmap_path=None):
+        self.app = create_app()
+        self.appcontext = self.app.app_context()
+        self.appcontext.push()
+        self._silence_sql_logging()
+        self.client = self.app.test_client()
+        self._ensure_asset_types()
+        self.secret = self._mint_admin_pat()
+        default = os.path.join(os.path.dirname(__file__), 'idmap.json')
+        self.ids = IdMap(idmap_path or default)
+        self.source = Source()
+        self.errors = []
+
+    def _silence_sql_logging(self):
+        import logging
+        logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)
+
+    def _ensure_asset_types(self):
+        """Register each plugin's AssetType + seeded subtypes. on_install (which
+        does this) is skipped when a DB is built with `flask plugin upgrade-all`
+        rather than a fresh `install`, so run it idempotently here - the create
+        routes 500 without the AssetType row."""
+        manager = self.app.extensions.get('plugin_manager')
+        if not manager:
+            return
+        for _name, plugin in manager.get_all_plugins().items():
+            try:
+                plugin.on_install(self.app)
+            except Exception:
+                pass
+        # A full import IS the setup, so mark it complete - otherwise the admin
+        # is bounced to the first-run wizard on every fresh import DB.
+        from shopdb.core.models import Setting
+        Setting.set('setup_complete', 'true', valuetype='boolean', category='site')
+        db.session.commit()
+
+    def _mint_admin_pat(self):
+        """Create (or reuse) an admin user and an unscoped admin PAT. Unscoped
+        because import mode + role gates require full owner authority
+        (the IMPORT-API wiki page). Returns the clear secret."""
+        from werkzeug.security import generate_password_hash
+        from shopdb.core.models import User, Role, ApiToken
+        admin = User.query.join(User.roles).filter(Role.rolename == 'admin').first()
+        if not admin:
+            role = Role.query.filter_by(rolename='admin').first() \
+                or Role(rolename='admin', description='Administrator')
+            db.session.add(role)
+            admin = User(username='importer', email='importer@localhost',
+                         passwordhash=generate_password_hash('x'), isactive=True)
+            admin.roles.append(role)
+            db.session.add(admin)
+            db.session.commit()
+        secret = ApiToken.generate_secret()
+        token = ApiToken(userid=admin.userid, name='wjf legacy import',
+                         tokenprefix=ApiToken.prefix_of(secret),
+                         tokenhash=ApiToken.hash_secret(secret), scopes=None)
+        db.session.add(token)
+        db.session.commit()
+        return secret
+
+    def _headers(self, import_mode):
+        # A PAT authenticates as Authorization: Bearer shopdb_pat_... (the
+        # before_request shim resolves it into a JWT identity). X-API-Key is for
+        # collector/service tokens, not personal tokens.
+        headers = {'Authorization': f'Bearer {self.secret}'}
+        if import_mode:
+            headers['X-Import-Mode'] = 'true'
+        return headers
+
+    def post(self, path, payload, import_mode=True):
+        """POST to an import endpoint. Returns (status, data). 409s are returned
+        (idempotency handled by the caller), other 4xx/5xx are recorded."""
+        resp = self.client.post(path, json=payload, headers=self._headers(import_mode))
+        body = resp.get_json() or {}
+        data = body.get('data', body)
+        if resp.status_code >= 400 and resp.status_code != 409:
+            self.errors.append((path, resp.status_code, payload, data))
+        return resp.status_code, data
+
+    def get(self, path):
+        resp = self.client.get(path, headers=self._headers(False))
+        body = resp.get_json() or {}
+        return resp.status_code, body.get('data', body)
+
+    def close(self):
+        self.ids.save()
+        self.appcontext.pop()
--- a/scripts/site_imports/wjf/run.py
+++ b/scripts/site_imports/wjf/run.py
@@ -0,0 +1,817 @@
+"""WJ classic-ASP -> flask reference import loader (site-specific).
+
+Run against a THROWAWAY import database:
+
+    DATABASE_URL='mysql+pymysql://root:PW@127.0.0.1:3306/shopdb_flask_import?charset=utf8mb4' \\
+        venv/bin/python -m scripts.site_imports.wjf.run --stages reference,employees
+
+Stages are idempotent (re-running resolves existing rows by natural key), so a
+crashed run resumes. See scripts/site_imports/wjf/harness.py for the contract
+this drives, and scratchpad/IMPORT-PLAN.md for the full mapping + decisions.
+
+Implemented: reference (vendors, businessunits, operatingsystems), employees.
+TODO stages (assets hub + dependents + network + usb) are stubbed - they need
+the machineid->assetid crosswalk this harness persists.
+"""
+
+import argparse
+
+from .harness import Harness
+
+
+def _id_of(row, idfield):
+    if not isinstance(row, dict):
+        return None
+    if row.get(idfield) is not None:
+        return row[idfield]
+    # asset-create responses nest the id at the top level (assetid) too
+    for key in (idfield, 'id', 'assetid'):
+        if row.get(key) is not None:
+            return row[key]
+    return None
+
+
+def _upsert(h, path, payload, unique_field, idfield, list_path=None):
+    """POST payload; on 409 resolve the existing row by its unique field.
+    Returns the new/existing id, or None."""
+    status, data = h.post(path, payload)
+    if status in (200, 201):
+        return _id_of(data, idfield)
+    if status == 409:
+        lookup = list_path or path
+        lookup += ('&' if '?' in lookup else '?') + 'per_page=10000'
+        _, rows = h.get(lookup)
+        items = rows if isinstance(rows, list) else rows.get('items', rows)
+        for row in (items or []):
+            if str(row.get(unique_field, '')).strip().lower() == \
+                    str(payload[unique_field]).strip().lower():
+                return _id_of(row, idfield)
+    return None
+
+
+# --- classic machinetype routing (per the resolved import decisions) ---------
+# measuringtools are the physical instruments (the CMM/gauge machine types). A
+# PC that DRIVES one (pctype CMM/Genspect/Keyence/Wax) is still a computer, not a
+# measuring tool - routing by pctype wrongly swept ~105 PCs into measuringtools.
+MEASURING_MTYPES = {3, 5, 8, 23, 47, 48}      # CMM, Wax, Eddy, Measuring, Inspection, Spline
+COMPUTER_MTYPES = {33, 20}                     # PC, Server
+NETWORK_MTYPES = {16, 17, 18, 19, 46}          # Access, IDF, Camera, Switch, Firewall
+SKIP_MTYPES = {15, 44}                          # Printer (printers table), USB (cmmc source)
+# LocationOnly is the islocationonly BIT, not machinetypeid=1: of the 158 type-1
+# rows only 24 carry the bit (real named areas). The other 134 are active,
+# modelled shop machines just left untyped - they must become assets, not
+# Locations.
+
+
+def _truthy_bit(value):
+    """MySQL bit(1) comes back from pymysql as bytes b'\\x00'/b'\\x01'."""
+    return value not in (0, None, False, b'\x00', b'', '0', '')
+# classic controllertypeid -> (vendor, model) hand split; 1=TBD dropped
+CONTROLLER_SPLIT = {2: ('Fanuc', '31i-MB'), 6: ('Fanuc', None),
+                    7: ('Okuma', None), 8: ('Makino', None)}
+
+
+def _route(machinetypeid, pctypeid, islocationonly=False):
+    if islocationonly:
+        return 'location'
+    if machinetypeid in SKIP_MTYPES:
+        return 'skip'
+    if machinetypeid in NETWORK_MTYPES:
+        return 'network'
+    if machinetypeid in MEASURING_MTYPES:
+        return 'measuringtool'
+    if machinetypeid in COMPUTER_MTYPES:
+        return 'computer'  # a metrology PC is still a PC (it CONTROLS a tool)
+    return 'machine'  # includes the 134 untyped (machinetypeid=1) shop machines
+
+
+def _modeltype_category(machinetypeid):
+    if machinetypeid in COMPUTER_MTYPES:
+        return 'PC'
+    if machinetypeid in NETWORK_MTYPES:
+        return 'Network'
+    if machinetypeid == 15:
+        return 'Printer'
+    return 'Equipment'
+
+
+def stage_reference(h):
+    """Seed the reference tier that later FKs resolve against, capturing
+    legacy-id -> new-id crosswalks."""
+    counts = {}
+
+    vendors = h.source.rows('shopdb_src',
+                            'SELECT vendorid, vendor FROM vendors WHERE isactive=1')
+    for v in vendors:
+        newid = _upsert(h, '/api/vendors', {'vendor': v['vendor']}, 'vendor', 'vendorid')
+        if newid:
+            h.ids.put('vendor', v['vendorid'], newid)
+    counts['vendors'] = h.ids.count('vendor')
+
+    bus = h.source.rows('shopdb_src',
+                        'SELECT businessunitid, businessunit FROM businessunits WHERE isactive=1')
+    for b in bus:
+        newid = _upsert(h, '/api/businessunits', {'businessunit': b['businessunit']},
+                        'businessunit', 'businessunitid')
+        if newid:
+            h.ids.put('businessunit', b['businessunitid'], newid)
+    counts['businessunits'] = h.ids.count('businessunit')
+
+    oses = h.source.rows('shopdb_src',
+                        'SELECT osid, operatingsystem FROM operatingsystems')
+    for o in oses:
+        name = (o['operatingsystem'] or '').strip()
+        if not name:
+            continue
+        newid = _upsert(h, '/api/operatingsystems', {'osname': name}, 'osname', 'osid')
+        if newid:
+            h.ids.put('os', o['osid'], newid)
+    counts['operatingsystems'] = h.ids.count('os')
+
+    return counts
+
+
+def stage_employees(h):
+    """Bulk-upsert the employee directory (selfhosted). Photos are deferred
+    (per the import decisions); occurrences are out of scope."""
+    from shopdb.core.models import Setting
+    Setting.set('employee_directory_mode', 'selfhosted', valuetype='string',
+                category='employees')
+    from shopdb.extensions import db
+    db.session.commit()
+
+    rows = h.source.rows('wjf_employees_src',
+                        'SELECT SSO, First_Name, Last_Name, Team, Role FROM employees')
+    lines = ['SSO,First_Name,Last_Name,Team,Role']
+    for r in rows:
+        vals = [str(r['SSO']), (r['First_Name'] or '').strip(),
+                (r['Last_Name'] or '').strip(), (r['Team'] or '').strip(),
+                (r['Role'] or '').strip()]
+        lines.append(','.join(v.replace(',', ' ') for v in vals))
+    status, data = h.post('/api/employees/directory/import', {'csv': '\n'.join(lines)})
+    return {'employees_posted': len(rows), 'result': data}
+
+
+def stage_catalog(h):
+    """Seed the type catalogs the asset routing + models depend on, capturing
+    crosswalks: modeltypes + per-plugin subtypes (from classic machinetypes),
+    computer subtypes (from pctype), controller vendor/model split, models."""
+    counts = {}
+    mtypes = h.source.rows('shopdb_src',
+                          'SELECT machinetypeid, machinetype FROM machinetypes')
+    for mt in mtypes:
+        name = (mt['machinetype'] or '').strip()
+        mtid = mt['machinetypeid']
+        # machinetypeid=1 (LocationOnly) is not a real machine subtype - the
+        # untyped shop machines that carry it import with a null subtype.
+        if not name or mtid == 1:
+            continue
+        # modeltype (types the models catalog)
+        modeltypeid = _upsert(h, '/api/modeltypes',
+                              {'modeltype': name, 'category': _modeltype_category(mtid)},
+                              'modeltype', 'modeltypeid')
+        if modeltypeid:
+            h.ids.put('modeltype', mtid, modeltypeid)
+        # per-plugin asset subtype, by route
+        route = _route(mtid, None)
+        if route == 'machine':
+            sid = _upsert(h, '/api/machines/types', {'machinetype': name},
+                          'machinetype', 'machinetypeid', '/api/machines/types')
+            if sid:
+                h.ids.put('machinesubtype', mtid, sid)
+        elif route == 'network':
+            sid = _upsert(h, '/api/network/types', {'networkdevicetype': name},
+                          'networkdevicetype', 'networkdevicetypeid', '/api/network/types')
+            if sid:
+                h.ids.put('networksubtype', mtid, sid)
+        elif route == 'measuringtool':
+            sid = _upsert(h, '/api/measuringtools/types', {'name': name},
+                          'name', 'measuringtooltypeid', '/api/measuringtools/types')
+            if sid:
+                h.ids.put('measuringsubtype', mtid, sid)
+    counts['modeltypes'] = h.ids.count('modeltype')
+
+    # computer subtypes from pctype
+    for pc in h.source.rows('shopdb_src', 'SELECT pctypeid, typename FROM pctype'):
+        name = (pc['typename'] or '').strip()
+        if not name:
+            continue
+        sid = _upsert(h, '/api/computers/types', {'computertype': name},
+                      'computertype', 'computertypeid', '/api/computers/types')
+        if sid:
+            h.ids.put('computertype', pc['pctypeid'], sid)
+    counts['computertypes'] = h.ids.count('computertype')
+
+    # controller split -> vendor + model, crosswalk controllertypeid -> both ids
+    for ctid, (vendor, model) in CONTROLLER_SPLIT.items():
+        vid = _upsert(h, '/api/vendors', {'vendor': vendor}, 'vendor', 'vendorid')
+        mid = None
+        if model and vid:
+            mid = _upsert(h, '/api/models', {'modelnumber': model, 'vendorid': vid},
+                          'modelnumber', 'modelnumberid')
+        h.ids.put('controllervendor', ctid, vid)
+        if mid:
+            h.ids.put('controllermodel', ctid, mid)
+
+    # models catalog (the only vendor source for machines)
+    for m in h.source.rows('shopdb_src',
+                          'SELECT modelnumberid, modelnumber, vendorid, machinetypeid '
+                          'FROM models WHERE isactive=1'):
+        number = (m['modelnumber'] or '').strip()
+        if not number:
+            continue
+        payload = {'modelnumber': number,
+                   'vendorid': h.ids.get('vendor', m['vendorid']),
+                   'modeltypeid': h.ids.get('modeltype', m['machinetypeid'])}
+        newid = _upsert(h, '/api/models', payload, 'modelnumber', 'modelnumberid')
+        if newid:
+            h.ids.put('model', m['modelnumberid'], newid)
+    counts['models'] = h.ids.count('model')
+    return counts
+
+
+def stage_assets(h):
+    """The hub: fan classic machines out to the right asset endpoint by type,
+    persisting the machineid -> assetid crosswalk everything downstream needs.
+    LocationOnly/printer/usb rows are skipped here (handled elsewhere / TODO)."""
+    # isactive=1 only: retired machines (classic keeps them as history) must not
+    # land as live assets. Every other stage already filters isactive=1; the
+    # assets hub is the one that leaked retired rows (incl. G-prefix hostnames).
+    machines = h.source.rows('shopdb_src', 'SELECT * FROM machines WHERE isactive=1')
+    seen_assetnumbers = set()
+    counts = {'computer': 0, 'machine': 0, 'network': 0, 'measuringtool': 0,
+              'skip_location': 0, 'skip_other': 0, 'skip_dup': 0, 'skip_9999': 0}
+
+    # Pre-seed assetnumbers already in the target (idempotent re-run).
+    for route in ('computers', 'machines', 'network', 'measuringtools'):
+        _, rows = h.get(f'/api/{route}?per_page=5000')
+        for row in (rows.get('items', rows) if isinstance(rows, dict) else rows) or []:
+            if isinstance(row, dict) and row.get('assetnumber'):
+                seen_assetnumbers.add(row['assetnumber'].strip().lower())
+
+    # On a duplicate machinenumber, first-wins - so process the richer asset
+    # types first (a PC carries installs/IP a bare untyped machine does not).
+    route_rank = {'computer': 0, 'measuringtool': 1, 'network': 2, 'machine': 3}
+    machines = sorted(machines, key=lambda m: route_rank.get(
+        _route(m['machinetypeid'], m['pctypeid'], _truthy_bit(m['islocationonly'])), 4))
+
+    for m in machines:
+        route = _route(m['machinetypeid'], m['pctypeid'],
+                       _truthy_bit(m['islocationonly']))
+        if route == 'location':
+            counts['skip_location'] += 1
+            continue
+        if route == 'skip':
+            counts['skip_other'] += 1
+            continue
+
+        raw = (m['machinenumber'] or '').strip()
+        if raw == '9999':
+            counts['skip_9999'] += 1
+            continue
+        assetnumber = raw or (m['hostname'] or '').strip()
+        if not assetnumber:
+            counts['skip_other'] += 1
+            continue
+        key = assetnumber.lower()
+        if key in seen_assetnumbers:
+            counts['skip_dup'] += 1
+            continue
+        seen_assetnumbers.add(key)
+
+        base = {
+            'assetnumber': assetnumber,
+            'name': (m['alias'] or m['hostname'] or '').strip() or None,
+            'serialnumber': (m['serialnumber'] or '').strip() or None,
+            'businessunitid': h.ids.get('businessunit', m['businessunitid']),
+            'mapx': m['mapleft'], 'mapy': m['maptop'],
+            'notes': m['machinenotes'],
+            'dateadded': str(m['dateadded']) if m['dateadded'] else None,
+            'modifieddate': str(m['lastupdated']) if m['lastupdated'] else None,
+        }
+        if route == 'computer':
+            path = '/api/computers'
+            payload = {**base, 'hostname': (m['hostname'] or '').strip() or None,
+                       'computertypeid': h.ids.get('computertype', m['pctypeid']),
+                       'osid': h.ids.get('os', m['osid']),
+                       'ipaddress': (m['ipaddress1'] or '').strip() or None}
+        elif route == 'network':
+            path = '/api/network'
+            payload = {**base, 'hostname': (m['hostname'] or m['fqdn'] or '').strip() or None,
+                       'networkdevicetypeid': h.ids.get('networksubtype', m['machinetypeid']),
+                       'vendorid': None,
+                       'ipaddress': (m['ipaddress1'] or '').strip() or None}
+        elif route == 'measuringtool':
+            path = '/api/measuringtools'
+            payload = {**base,
+                       'measuringtooltypeid': h.ids.get('measuringsubtype', m['machinetypeid'])}
+        else:  # machine
+            path = '/api/machines'
+            payload = {**base, 'machinetypeid': h.ids.get('machinesubtype', m['machinetypeid']),
+                       'modelnumberid': h.ids.get('model', m['modelnumberid']),
+                       'vendorid': None,
+                       'controllervendorid': h.ids.get('controllervendor', m['controllertypeid']),
+                       'controllermodelid': h.ids.get('controllermodel', m['controllertypeid'])}
+
+        status, data = h.post(path, payload)
+        if status in (200, 201):
+            assetid = _id_of(data, 'assetid')
+            if assetid:
+                h.ids.put('asset', m['machineid'], assetid)
+                counts[route] += 1
+    return counts
+
+
+def stage_printers(h):
+    """Printers come from the printers TABLE (not the machines hub). assetnumber
+    is synthesized PRN-{printerid} (no machinenumber). The printer's IP folds
+    into a Communication via the create route. Location resolves from the host
+    machineid when it is one of the LocationOnly rows. Skips inactive."""
+    made = 0
+    for p in h.source.rows('shopdb_src',
+                          'SELECT printerid, modelid, printerwindowsname, printercsfname, '
+                          'serialnumber, fqdn, ipaddress, machineid, maptop, mapleft, iscsf, '
+                          'installpath, printernotes, printerpin FROM printers WHERE isactive=1'):
+        payload = {
+            'assetnumber': f"PRN-{p['printerid']}",
+            'name': (p['printerwindowsname'] or '').strip() or None,
+            'serialnumber': (p['serialnumber'] or '').strip() or None,
+            'hostname': (p['fqdn'] or '').strip() or None,
+            'windowsname': (p['printerwindowsname'] or '').strip() or None,
+            'sharename': (p['printercsfname'] or '').strip() or None,
+            'ipaddress': (p['ipaddress'] or '').strip() or None,
+            'iscsf': _truthy_bit(p['iscsf']),
+            'installpath': p['installpath'], 'pin': p['printerpin'],
+            'notes': p['printernotes'], 'mapx': p['mapleft'], 'mapy': p['maptop'],
+            'modelnumberid': h.ids.get('model', p['modelid']),
+            'locationid': h.ids.get('location', p['machineid']),
+        }
+        status, _ = h.post('/api/printers', payload)
+        if status in (200, 201):
+            made += 1
+    return {'printers': made}
+
+
+# classic pctype -> the measuring instrument that PC drives
+METROLOGY_PCTYPE_TOOL = {5: 'CMM', 6: 'Form Tracer', 7: 'Vision System', 8: 'Genspect'}
+
+
+def stage_metrology(h):
+    """Metrology PCs (CMM/Genspect/Keyence/Wax) are computers that CONTROL an
+    instrument. Classic has no separate tool row, so synthesize a measuring-tool
+    asset per metrology PC and a Controls relationship (PC -> tool), mirroring
+    what the runtime collector does. Runs after the asset hub (needs the PC's
+    assetid from the crosswalk)."""
+    controls_id = _upsert(h, '/api/assets/relationshiptypes',
+                          {'relationshiptype': 'Controls'}, 'relationshiptype',
+                          'relationshiptypeid', '/api/assets/relationshiptypes')
+    made = linked = 0
+    for m in h.source.rows('shopdb_src',
+                          'SELECT machineid, alias, hostname, machinenumber, businessunitid, '
+                          'mapleft, maptop, pctypeid FROM machines '
+                          'WHERE machinetypeid=33 AND pctypeid IN (5,6,7,8) '
+                          'AND isactive=1 '
+                          'AND (islocationonly IS NULL OR islocationonly=0)'):
+        pc_assetid = h.ids.get('asset', m['machineid'])
+        if not pc_assetid:
+            continue  # PC not imported (dup-skipped etc.)
+        toolname = METROLOGY_PCTYPE_TOOL[m['pctypeid']]
+        typeid = _upsert(h, '/api/measuringtools/types', {'name': toolname},
+                         'name', 'measuringtooltypeid', '/api/measuringtools/types')
+        pcname = (m['alias'] or m['hostname'] or m['machinenumber'] or '').strip()
+        status, data = h.post('/api/measuringtools', {
+            'assetnumber': f"MT-{m['machineid']}",
+            'name': (f"{pcname} {toolname}").strip() or toolname,
+            'measuringtooltypeid': typeid,
+            'businessunitid': h.ids.get('businessunit', m['businessunitid']),
+            'mapx': m['mapleft'], 'mapy': m['maptop']})
+        if status not in (200, 201):
+            continue
+        tool_assetid = _id_of(data, 'assetid')
+        made += 1
+        if tool_assetid and controls_id:
+            h.ids.put('metrologytool', m['machineid'], tool_assetid)
+            st, _ = h.post('/api/assets/relationships', {
+                'sourceassetid': pc_assetid, 'targetassetid': tool_assetid,
+                'relationshiptypeid': controls_id})
+            if st in (200, 201):
+                linked += 1
+    return {'measuringtools_synth': made, 'controls_links': linked}
+
+
+def stage_communications(h):
+    """Fold each asset's primary IP from the source communications table. There
+    is no bulk-communications endpoint, so this is one of the plan's documented
+    direct-ORM gaps (the API only takes a primary IP on asset create/update)."""
+    from shopdb.core.models import Communication, CommunicationType
+    from shopdb.extensions import db
+    ip_type = CommunicationType.query.filter_by(comtype='IP').first()
+    rows = h.source.rows('shopdb_src',
+                        'SELECT machineid, address FROM communications '
+                        'WHERE comstypeid=1 AND isprimary=1')
+    added = 0
+    for r in rows:
+        assetid = h.ids.get('asset', r['machineid'])
+        ip = (r['address'] or '').strip()
+        if not (assetid and ip and ip_type):
+            continue
+        if Communication.query.filter_by(assetid=assetid, isprimary=True).first():
+            continue
+        db.session.add(Communication(assetid=assetid, comtypeid=ip_type.comtypeid,
+                                     ipaddress=ip, isprimary=True))
+        added += 1
+    db.session.commit()
+    return {'primary_ips': added}
+
+
+def stage_applications(h):
+    """Support teams + applications catalog + versions + installs. Skips
+    inactive rows (decision); dedup of colliding app names is automatic via the
+    unique-appname 409-resolve."""
+    from plugins.computers.models import Computer
+    counts = {}
+
+    for t in h.source.rows('shopdb_src',
+                          'SELECT supporteamid, teamname, teamurl FROM supportteams WHERE isactive=1'):
+        name = (t['teamname'] or '').strip()
+        if not name:
+            continue
+        newid = _upsert(h, '/api/supportteams', {'teamname': name, 'teamurl': t['teamurl']},
+                        'teamname', 'supportteamid', '/api/supportteams')
+        if newid:
+            h.ids.put('supportteam', t['supporteamid'], newid)
+    counts['supportteams'] = h.ids.count('supportteam')
+
+    for a in h.source.rows('shopdb_src',
+                          'SELECT appid, appname, appdescription, supportteamid, '
+                          'isinstallable, applicationnotes, installpath, applicationlink, '
+                          'documentationpath FROM applications WHERE isactive=1'):
+        name = (a['appname'] or '').strip()
+        if not name:
+            continue
+        payload = {'appname': name, 'appdescription': a['appdescription'],
+                   'supportteamid': h.ids.get('supportteam', a['supportteamid']),
+                   'isinstallable': bool(a['isinstallable']),
+                   'applicationnotes': a['applicationnotes'], 'installpath': a['installpath'],
+                   'applicationlink': a['applicationlink'],
+                   'documentationpath': a['documentationpath']}
+        newid = _upsert(h, '/api/applications', payload, 'appname', 'appid',
+                        '/api/applications')
+        if newid:
+            h.ids.put('app', a['appid'], newid)
+    counts['applications'] = h.ids.count('app')
+
+    for v in h.source.rows('shopdb_src',
+                          'SELECT appversionid, appid, version, releasedate, notes, dateadded '
+                          'FROM appversions WHERE isactive=1'):
+        appid = h.ids.get('app', v['appid'])
+        ver = (v['version'] or '').strip()
+        if not (appid and ver):
+            continue
+        status, data = h.post(f'/api/applications/{appid}/versions',
+                              {'version': ver, 'releasedate': str(v['releasedate']) if v['releasedate'] else None,
+                               'notes': v['notes'], 'dateadded': str(v['dateadded']) if v['dateadded'] else None})
+        if status in (200, 201):
+            h.ids.put('appversion', v['appversionid'], _id_of(data, 'appversionid'))
+    counts['appversions'] = h.ids.count('appversion')
+
+    installed = 0
+    for i in h.source.rows('shopdb_src',
+                          'SELECT appid, appversionid, machineid FROM installedapps WHERE isactive=1'):
+        assetid = h.ids.get('asset', i['machineid'])
+        appid = h.ids.get('app', i['appid'])
+        if not (assetid and appid):
+            continue
+        computer = Computer.query.filter_by(assetid=assetid).first()
+        if not computer:
+            continue
+        payload = {'appid': appid}
+        vid = h.ids.get('appversion', i['appversionid'])
+        if vid:
+            payload['appversionid'] = vid
+        status, _ = h.post(f'/api/computers/{computer.computerid}/apps', payload)
+        if status in (200, 201):
+            installed += 1
+    counts['installs'] = installed
+    return counts
+
+
+def stage_warranties(h):
+    """Warranties -> POST /api/warranty, linked to the asset via the crosswalk.
+    The source has no vendor; hardcode Dell (decision)."""
+    linked = 0
+    for w in h.source.rows('shopdb_src',
+                          'SELECT machineid, warrantyname, enddate, servicelevel, dateadded '
+                          'FROM warranties'):
+        assetid = h.ids.get('asset', w['machineid'])
+        if not assetid:
+            continue
+        payload = {'vendor': 'Dell', 'provider': 'manual',
+                   'servicelevel': (w['servicelevel'] or '').strip() or None,
+                   'enddate': str(w['enddate']) if w['enddate'] else None,
+                   'assetids': [assetid],
+                   'dateadded': str(w['dateadded']) if w['dateadded'] else None}
+        status, _ = h.post('/api/warranty', payload)
+        if status in (200, 201):
+            linked += 1
+    return {'warranties': linked}
+
+
+def stage_notifications(h):
+    """Notification types + notifications."""
+    counts = {}
+    for t in h.source.rows('shopdb_src',
+                          'SELECT notificationtypeid, typename, typecolor FROM notificationtypes WHERE isactive=1'):
+        name = (t['typename'] or '').strip()
+        if not name:
+            continue
+        newid = _upsert(h, '/api/notifications/types',
+                        {'typename': name, 'typecolor': (t['typecolor'] or '').strip() or None},
+                        'typename', 'notificationtypeid', '/api/notifications/types')
+        if newid:
+            h.ids.put('notificationtype', t['notificationtypeid'], newid)
+    counts['notificationtypes'] = h.ids.count('notificationtype')
+
+    # SSO -> "First Last" from the employee source, so recognition/training
+    # notifications display a name, not a bare SSO (the model shows employeename
+    # or falls back to employeesso).
+    ssoname = {}
+    for e in h.source.rows('wjf_employees_src',
+                          'SELECT SSO, First_Name, Last_Name FROM employees'):
+        full = f"{(e['First_Name'] or '').strip()} {(e['Last_Name'] or '').strip()}".strip()
+        if full:
+            ssoname[str(e['SSO']).strip()] = full
+
+    def _names(employeesso):
+        if not employeesso:
+            return None
+        parts = [ssoname.get(s.strip()) for s in str(employeesso).split(',') if s.strip()]
+        parts = [p for p in parts if p]
+        return ', '.join(parts) or None
+
+    made = 0
+    for n in h.source.rows('shopdb_src',
+                          'SELECT notificationtypeid, businessunitid, appid, notification, '
+                          'starttime, endtime, ticketnumber, link, isshopfloor, employeesso '
+                          'FROM notifications WHERE isactive=1'):
+        text = (n['notification'] or '').strip()
+        if not text:
+            continue
+        endtime = str(n['endtime']) if n['endtime'] else None
+        if endtime and endtime.startswith('2099'):
+            endtime = None
+        payload = {'notification': text,
+                   'notificationtypeid': h.ids.get('notificationtype', n['notificationtypeid']),
+                   'businessunitid': h.ids.get('businessunit', n['businessunitid']),
+                   'appid': h.ids.get('app', n['appid']),
+                   'starttime': str(n['starttime']) if n['starttime'] else None,
+                   'endtime': endtime, 'ticketnumber': n['ticketnumber'], 'link': n['link'],
+                   'isshopfloor': bool(n['isshopfloor']), 'employeesso': n['employeesso'],
+                   'employeename': _names(n['employeesso'])}
+        status, _ = h.post('/api/notifications', payload)
+        if status in (200, 201):
+            made += 1
+    counts['notifications'] = made
+    return counts
+
+
+def stage_knowledgebase(h):
+    """KB articles. appid resolves through the applications name map (topics
+    folded into applications)."""
+    made = 0
+    for k in h.source.rows('shopdb_src',
+                          'SELECT shortdescription, keywords, appid, linkurl, notes '
+                          'FROM knowledgebase WHERE isactive=1'):
+        title = (k['shortdescription'] or '').strip()
+        url = (k['linkurl'] or '').strip()
+        if not (title and url):
+            continue
+        payload = {'shortdescription': title, 'linkurl': url,
+                   'keywords': k['keywords'], 'notes': k['notes'],
+                   'appid': h.ids.get('app', k['appid'])}
+        status, _ = h.post('/api/knowledgebase', payload)
+        if status in (200, 201):
+            made += 1
+    return {'kb_articles': made}
+
+
+def stage_locations(h):
+    """The 24 islocationonly rows -> core Locations. Crosswalk machineid ->
+    locationid (relationships can point at them)."""
+    made = 0
+    for m in h.source.rows('shopdb_src',
+                          'SELECT machineid, machinenumber, alias FROM machines '
+                          'WHERE machinetypeid=1 AND islocationonly=1 AND isactive=1'):
+        name = (m['alias'] or m['machinenumber'] or '').strip()
+        if not name:
+            continue
+        newid = _upsert(h, '/api/locations', {'locationname': name},
+                        'locationname', 'locationid', '/api/locations')
+        if newid:
+            h.ids.put('location', m['machineid'], newid)
+            made += 1
+    return {'locations': made}
+
+
+def stage_relationships(h):
+    """Active machinerelationships -> asset relationships via the crosswalk.
+    Edges whose endpoints did not become assets (locations/skipped/dups) drop.
+    Dedups on the (source, target, type) triple."""
+    # Seed relationship types (case-insensitive collation folds Controls onto
+    # the seeded 'controls', etc), capturing the crosswalk. A "Controlled By"
+    # type is the reverse of "Controls" - flag it so edges get flipped to the
+    # forward direction instead of importing a redundant inverse type.
+    reverse_typeids = set()
+    controls_newid = None
+    for t in h.source.rows('shopdb_src',
+                          'SELECT relationshiptypeid, relationshiptype FROM relationshiptypes WHERE isactive=1'):
+        name = (t['relationshiptype'] or '').strip()
+        if not name:
+            continue
+        lname = name.lower()
+        if 'controlled by' in lname:
+            reverse_typeids.add(t['relationshiptypeid'])
+            continue  # do not create an inverse type; edges map to Controls
+        newid = _upsert(h, '/api/assets/relationshiptypes', {'relationshiptype': name},
+                        'relationshiptype', 'relationshiptypeid', '/api/assets/relationshiptypes')
+        if newid:
+            h.ids.put('relationshiptype', t['relationshiptypeid'], newid)
+            if lname == 'controls':
+                controls_newid = newid
+
+    seen = set()
+    made = dropped = 0
+    for r in h.source.rows('shopdb_src',
+                          'SELECT machineid, related_machineid, relationshiptypeid '
+                          'FROM machinerelationships WHERE isactive=1'):
+        source = h.ids.get('asset', r['machineid'])
+        target = h.ids.get('asset', r['related_machineid'])
+        # "Controlled By" edges flip to the forward Controls direction.
+        if r['relationshiptypeid'] in reverse_typeids:
+            source, target = target, source
+            typeid = controls_newid
+        else:
+            typeid = h.ids.get('relationshiptype', r['relationshiptypeid'])
+        if not (source and target and typeid) or source == target:
+            dropped += 1
+            continue
+        key = (source, target, typeid)
+        if key in seen:
+            continue
+        seen.add(key)
+        status, _ = h.post('/api/assets/relationships',
+                          {'sourceassetid': source, 'targetassetid': target,
+                           'relationshiptypeid': typeid})
+        if status in (200, 201):
+            made += 1
+    return {'relationships': made, 'dropped_unresolved': dropped}
+
+
+def stage_subnets(h):
+    """Subnets + VLANs. Classic cidr is the suffix only; full CIDR =
+    INET_NTOA(ipstart)+suffix. VLANs are lookup-or-create by number; duplicate
+    full CIDRs are first-wins (409 skip)."""
+    import socket
+    import struct
+    subtypes = {s['subnettypeid']: s['subnettype'] for s in
+                h.source.rows('shopdb_src', 'SELECT subnettypeid, subnettype FROM subnettypes')}
+    made = 0
+    for s in h.source.rows('shopdb_src',
+                          'SELECT vlan, description, ipstart, cidr, subnettypeid FROM subnets WHERE isactive=1'):
+        if s['ipstart'] is None or not s['cidr']:
+            continue
+        network = socket.inet_ntoa(struct.pack('>I', s['ipstart'] & 0xffffffff))
+        suffix = str(s['cidr']).strip()
+        cidr = network + (suffix if suffix.startswith('/') else '/' + suffix)
+        vlanid = None
+        if s['vlan']:
+            vlanid = _upsert(h, '/api/network/vlans',
+                             {'vlannumber': s['vlan'], 'name': f"VLAN {s['vlan']}"},
+                             'vlannumber', 'vlanid', '/api/network/vlans')
+        name = (s['description'] or '').strip() or cidr
+        payload = {'name': name, 'cidr': cidr, 'networkaddress': network,
+                   'vlanid': vlanid, 'subnettype': subtypes.get(s['subnettypeid'])}
+        status, _ = h.post('/api/network/subnets', payload)
+        if status in (200, 201):
+            made += 1
+    return {'subnets': made}
+
+
+def stage_usb(h):
+    """CMMC USB kiosk: devices + the check-in/out log paired into checkouts.
+    Requires selfhosted mode. cmmc users are directory data, not usb rows."""
+    from shopdb.core.models import Setting
+    from shopdb.extensions import db
+    Setting.set('usb_directory_mode', 'selfhosted', valuetype='string', category='usb')
+    db.session.commit()
+
+    devices = 0
+    for d in h.source.rows('cmmc_usb_src', 'SELECT device_id, device_desc, device_owner, locker_location FROM devices'):
+        did = (d['device_id'] or '').strip()
+        if not did or did.lower() in ('test', 'test123'):
+            continue
+        status, _ = h.post('/api/usb', {'device_id': did, 'device_desc': d['device_desc'],
+                                        'device_owner': d['device_owner'],
+                                        'locker_location': d['locker_location']})
+        if status in (200, 201, 409):
+            devices += 1
+
+    # Replay the log chronologically per device, tracking open state so we only
+    # check out an available device and check in an out one (the source log has
+    # unpaired entries - a checkin with no open checkout would 400).
+    events = 0
+    checked_out = {}
+    log = h.source.rows('cmmc_usb_src',
+                       'SELECT device_id, badge_number, action, timestamp, locker_location '
+                       'FROM checkinoutlog ORDER BY device_id, timestamp')
+    for e in log:
+        did = (e['device_id'] or '').strip()
+        badge = str(e['badge_number'] or '').strip()
+        action = (e['action'] or '').strip().lower()
+        if not (did and badge):
+            continue
+        when = str(e['timestamp']) if e['timestamp'] else None
+        if 'out' in action and not checked_out.get(did):
+            status, _ = h.post(f'/api/usb/{did}/checkout',
+                              {'badge': badge, 'checkouttime': when,
+                               'locker_location': e['locker_location']})
+            if status in (200, 201):
+                checked_out[did] = True
+                events += 1
+        elif 'in' in action and checked_out.get(did):
+            status, _ = h.post(f'/api/usb/{did}/checkin', {'badge': badge, 'checkintime': when})
+            if status in (200, 201):
+                checked_out[did] = False
+                events += 1
+    return {'usb_devices': devices, 'usb_events': events}
+
+
+def stage_verify(h):
+    """Row-count audit: source active count vs target count for each entity."""
+    checks = [
+        ('vendors', 'shopdb_src', 'SELECT COUNT(*) c FROM vendors WHERE isactive=1', 'vendors'),
+        ('assets', 'shopdb_src', 'SELECT COUNT(*) c FROM machines WHERE isactive=1', 'assets'),
+        ('applications', 'shopdb_src', 'SELECT COUNT(*) c FROM applications WHERE isactive=1', 'applications'),
+        ('knowledgebase', 'shopdb_src', 'SELECT COUNT(*) c FROM knowledgebase WHERE isactive=1', 'knowledgebase'),
+        ('employees', 'wjf_employees_src', 'SELECT COUNT(*) c FROM employees', 'directoryemployees'),
+    ]
+    from shopdb.extensions import db
+    from sqlalchemy import text
+    report = {}
+    for label, sdb, sql, target_table in checks:
+        src = h.source.rows(sdb, sql)[0]['c']
+        tgt = db.session.execute(text(f'SELECT COUNT(*) FROM {target_table}')).scalar()
+        report[label] = f'source~{src} target={tgt}'
+    return report
+
+
+STAGES = {
+    'reference': stage_reference,
+    'employees': stage_employees,
+    'catalog': stage_catalog,
+    'assets': stage_assets,
+    'locations': stage_locations,
+    'printers': stage_printers,
+    'metrology': stage_metrology,
+    'communications': stage_communications,
+    'applications': stage_applications,
+    'warranties': stage_warranties,
+    'notifications': stage_notifications,
+    'knowledgebase': stage_knowledgebase,
+    'relationships': stage_relationships,
+    'subnets': stage_subnets,
+    'usb': stage_usb,
+    'verify': stage_verify,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--stages',
+        default='reference,employees,catalog,assets,locations,printers,'
+                'metrology,communications,applications,warranties,notifications,'
+                'knowledgebase,relationships,subnets,usb,verify',
+        help='comma list of stages to run')
+    args = parser.parse_args()
+
+    h = Harness()
+    print(f'== WJ import loader (target: {h.app.config["SQLALCHEMY_DATABASE_URI"].split("/")[-1]}) ==')
+    try:
+        for name in args.stages.split(','):
+            name = name.strip()
+            if name not in STAGES:
+                print(f'  skip unknown stage: {name}')
+                continue
+            result = STAGES[name](h)
+            print(f'  [{name}] {result}')
+        if h.errors:
+            print(f'\n  {len(h.errors)} endpoint error(s):')
+            for path, code, _payload, data in h.errors[:10]:
+                print(f'    {code} {path}: {data}')
+    finally:
+        h.close()
+
+
+if __name__ == '__main__':
+    main()
--- a/scripts/test-external-plugin.sh
+++ b/scripts/test-external-plugin.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# test-external-plugin.sh
+#
+# Verify an out-of-tree shopdb-flask plugin against a pinned framework build.
+# Stand up a throwaway copy of the framework, drop the plugin into
+# plugins/<name>/ as a symlink (the mechanism ADR-003 documents for sister
+# sites), and run the framework contract tests plus the plugin's own tests.
+# Nonzero exit means the plugin is not compatible with that framework ref.
+#
+# Two modes:
+#
+#   CI / remote (default): clone the framework at FRAMEWORK_REF from
+#     FRAMEWORK_URL, build a fresh venv, pip install requirements. Needs
+#     network access to git and PyPI.
+#
+#   Local / offline: set LOCAL_FRAMEWORK to a framework checkout on disk.
+#     The script exports that checkout at HEAD with `git archive` (no network)
+#     and reuses the checkout's existing venv, so it runs with no internet.
+#
+# Inputs (env var, or positional):
+#   PLUGIN_DIR      ($1)  path to the plugin directory (holds manifest.json). Required.
+#   FRAMEWORK_REF   ($2)  git ref to test against in CI mode. Default: main.
+#   FRAMEWORK_URL         framework git URL for CI mode.
+#                         Default: https://<git-host>/ge-aerospace/shopdb-flask.git
+#   LOCAL_FRAMEWORK       path to an existing framework checkout. Set it to run offline.
+#
+# Examples:
+#   # CI: test the plugin in the current repo against a pinned tag
+#   PLUGIN_DIR=. FRAMEWORK_REF=v0.5.0 scripts/test-external-plugin.sh
+#
+#   # Offline: test a bundled plugin against this checkout, no network
+#   LOCAL_FRAMEWORK=. PLUGIN_DIR=plugins/warranty scripts/test-external-plugin.sh
+#
+# Coverage: a symlinked plugin is discovered and loaded by the plugin loader
+# (loadability, manifest validity, core_version range) AND scanned by the
+# import-surface contract test - the scanner resolves each plugin dir before
+# walking, since bare Path.rglob does not descend symlinks. Optionally keep an
+# equivalent import-surface assertion in your own tests/ so violations fail
+# in your repo's CI too (see the PLUGIN-EXTERNAL-REPO wiki page).
+
+set -eu
+
+PLUGIN_DIR="${PLUGIN_DIR:-${1:-}}"
+FRAMEWORK_REF="${FRAMEWORK_REF:-${2:-main}}"
+FRAMEWORK_URL="${FRAMEWORK_URL:-https://<git-host>/ge-aerospace/shopdb-flask.git}"
+LOCAL_FRAMEWORK="${LOCAL_FRAMEWORK:-}"
+
+if [ -z "$PLUGIN_DIR" ]; then
+  echo "ERROR: PLUGIN_DIR is required (env var or first argument)." >&2
+  exit 2
+fi
+if [ ! -f "$PLUGIN_DIR/manifest.json" ]; then
+  echo "ERROR: $PLUGIN_DIR has no manifest.json - not a plugin directory." >&2
+  exit 2
+fi
+
+# Absolute path so the symlink still resolves after we cd into the temp tree.
+PLUGIN_ABS="$(cd "$PLUGIN_DIR" && pwd)"
+# Plugin name: prefer the manifest name, fall back to the directory basename.
+PLUGIN_NAME="$(sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$PLUGIN_ABS/manifest.json" | head -n1)"
+if [ -z "$PLUGIN_NAME" ]; then
+  PLUGIN_NAME="$(basename "$PLUGIN_ABS")"
+fi
+
+WORKDIR="$(mktemp -d)"
+cleanup() { rm -rf "$WORKDIR"; }
+trap cleanup EXIT
+
+FRAMEWORK="$WORKDIR/framework"
+mkdir -p "$FRAMEWORK"
+
+if [ -n "$LOCAL_FRAMEWORK" ]; then
+  echo "==> Local mode: exporting framework from $LOCAL_FRAMEWORK (HEAD)"
+  LOCAL_ABS="$(cd "$LOCAL_FRAMEWORK" && pwd)"
+  git -C "$LOCAL_ABS" archive HEAD | tar -x -C "$FRAMEWORK"
+  # Reuse the checkout's venv so no pip and no network are needed.
+  if [ -x "$LOCAL_ABS/venv/bin/python" ]; then
+    PYTHON="$LOCAL_ABS/venv/bin/python"
+  elif [ -x "$LOCAL_ABS/.venv/bin/python" ]; then
+    PYTHON="$LOCAL_ABS/.venv/bin/python"
+  else
+    echo "ERROR: no venv found under $LOCAL_ABS (looked for venv/ and .venv/)." >&2
+    exit 2
+  fi
+else
+  echo "==> CI mode: cloning $FRAMEWORK_URL @ $FRAMEWORK_REF"
+  git clone --depth 1 --branch "$FRAMEWORK_REF" "$FRAMEWORK_URL" "$FRAMEWORK"
+  echo "==> Creating venv and installing requirements"
+  python3 -m venv "$WORKDIR/venv"
+  PYTHON="$WORKDIR/venv/bin/python"
+  "$PYTHON" -m pip install --upgrade pip >/dev/null
+  "$PYTHON" -m pip install -r "$FRAMEWORK/requirements.txt"
+fi
+
+echo "==> Linking plugin '$PLUGIN_NAME' into framework plugins/"
+rm -rf "$FRAMEWORK/plugins/$PLUGIN_NAME"
+ln -s "$PLUGIN_ABS" "$FRAMEWORK/plugins/$PLUGIN_NAME"
+
+# Drop any exported registry so the loader re-discovers and enables the plugin
+# (the test conftest seeds a fresh registry from plugins/*/manifest.json).
+rm -f "$FRAMEWORK/instance/plugins.json"
+
+RC=0
+
+echo "==> Running framework contract tests"
+( cd "$FRAMEWORK" && "$PYTHON" -m pytest tests/test_plugin_contract.py -q ) || RC=1
+
+if [ -d "$PLUGIN_ABS/tests" ]; then
+  echo "==> Running plugin's own tests"
+  ( cd "$FRAMEWORK" && "$PYTHON" -m pytest "$PLUGIN_ABS/tests" -q ) || RC=1
+else
+  echo "==> Plugin has no tests/ directory - skipping plugin test step"
+fi
+
+if [ "$RC" -eq 0 ]; then
+  echo "==> PASS: plugin '$PLUGIN_NAME' is compatible with framework ref '$FRAMEWORK_REF'"
+else
+  echo "==> FAIL: plugin '$PLUGIN_NAME' - see output above" >&2
+fi
+exit "$RC"
				`@@ -0,0 +1 @@`
				`"""Data migration scripts for VBScript ShopDB to Flask migration."""`