diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f24c78..d171111 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,34 @@ ADR-007 and ADR-002. ## [Unreleased] +### Added + +- Import mode: a complete, idempotent HTTP migration surface so a script or LLM + can import the classic ASP shopdb through the API alone (no direct DB writes). + - Contract surface (plugin contract bumped 0.7.0 -> 0.8.0, additive): new + `shopdb.api` helpers `apply_import_timestamps`, `import_mode_active`, + `parse_import_datetime` in `shopdb/utils/import_mode.py`. When the caller is + an admin AND sends header `X-Import-Mode: true`, create/update endpoints + accept optional `createddate` / `modifieddate` (ISO or legacy + `YYYY-MM-DD HH:MM:SS`, naive-UTC) and preserve them instead of stamping now. + Non-admin or missing header: the fields are ignored exactly as before. + Wired into every timestamped import target: assets (all five type plugins), + vendors, models, modeltypes, businessunits, locations, operating systems, + applications, knowledge base, USB devices, and asset relationships. + - Natural-key exact-match lookup filters for the documented + lookup-then-upsert idempotency recipe: `assetnumber` on all five asset + plugin list endpoints; `vendor`, `modelnumber`, `modeltype`, + `businessunit`, `locationname`, `osname`/`osversion`, `appname`, + knowledge base `linkurl`/`shortdescription`, warranty `servicetag`/`vendor`, + and notification `ticketnumber`. + - Backdated event history: in import mode the selfhosted USB checkout/checkin + endpoints accept optional `checkouttime` / `checkintime` overrides so + migrated `usbcheckouts` rows keep their real event times. + - New operator manual `docs/IMPORT-API.md` grounded in the real `prodscratch` + legacy schema: order of operations, a full table-by-table mapping, honest + no-target list with dispositions, a worked idempotent Python importer, and + row-count parity checks. + ## [0.6.0] - 2026-07-11 ### Added diff --git a/docs/IMPORT-API.md b/docs/IMPORT-API.md new file mode 100644 index 0000000..2d39fdd --- /dev/null +++ b/docs/IMPORT-API.md @@ -0,0 +1,487 @@ +# Import API: migrating the classic ASP shopdb through HTTP alone + +This is the operator manual for importing the legacy Classic-ASP shopdb database +(`prodscratch` on the dev MySQL container) into shopdb-flask using ONLY the HTTP +API. No direct writes to the `shopdb_flask` database are needed or wanted: every +row is created through a documented endpoint so authorization, validation, +auditing, and plugin hooks all run exactly as they do for a human operator. + +An LLM or a plain Python script can run the whole migration from this document. + +Contents: + +1. [Prerequisites](#1-prerequisites) +2. [Order of operations](#2-order-of-operations) +3. [Full table-by-table mapping](#3-full-table-by-table-mapping) +4. [Tables with no target yet](#4-tables-with-no-target-yet) +5. [Idempotency recipe and a worked importer](#5-idempotency-recipe-and-a-worked-importer) +6. [Verification: row-count parity](#6-verification-row-count-parity) + +--- + +## 1. Prerequisites + +### Dev URLs + +- Flask API: `http://localhost:5001` +- All import calls target `/api/...` on that host. + +### Admin token + +Every write needs a JWT, and import mode additionally needs an admin. Get one: + +```bash +curl -s http://localhost:5001/api/auth/login \ + -H 'Content-Type: application/json' \ + -d '{"username":"","password":""}' | jq -r '.data.access_token' +``` + +Send it on every request as `Authorization: Bearer `. + +### Import mode: the `X-Import-Mode` header + +By default the server stamps `createddate`/`modifieddate` to "now" on every +create and update, which would erase a migrated row's real history. To preserve +it, send the request header: + +``` +X-Import-Mode: true +``` + +When (and only when) the caller is an admin AND that header is present: + +- create/update endpoints on timestamped entities accept optional + `createddate` and `modifieddate` fields in the JSON body and store them + verbatim (naive UTC). Both `2020-01-05T12:00:00` (ISO) and the legacy + `2020-01-05 12:00:00` (MySQL) forms are parsed. A bare `2020-01-05` works too. +- the selfhosted USB checkout/checkin endpoints accept optional `checkouttime` + and `checkintime` overrides so historical events keep their real timestamps. + +Without the header, or for a non-admin caller, those fields are silently ignored +and the server behaves exactly as it does normally. This is enforced centrally +by `shopdb/utils/import_mode.py` (`import_mode_active`, `apply_import_timestamps`, +`parse_import_datetime`), exposed on the plugin contract surface `shopdb.api`. + +Timestamped entities that honor `createddate`/`modifieddate`: assets (all five +type plugins), vendors, models, modeltypes, businessunits, locations, operating +systems, applications, knowledge base, USB devices, asset relationships. + +Entities that carry history in domain fields instead (createddate passthrough is +a no-op there, by design): notifications (`starttime`/`endtime`), warranties +(`startdate`/`enddate`/`lastcheckeddate`). Set those fields directly in the +payload; they are already accepted. + +### Reference data seed + +Before importing, seed the reference tables that have no CRUD endpoint of their +own (communication types such as IP/Serial/USB, default statuses, canonical +relationship types, permissions, settings): + +```bash +flask seed permissions +flask seed settings +flask seed reference-data +``` + +`communicationtypes` (the target of legacy `comstypes`) is populated here, so +the primary-IP mapping below can resolve `comtype='IP'`. + +--- + +## 2. Order of operations + +Import in dependency order so foreign keys always resolve. Each step is a +lookup-then-upsert loop (see section 5); rerunning any step is safe. + +1. **Reference / lookup data first** + 1. Vendors (`vendors`) + 2. Model types (`modeltypes`) - from legacy `machinetypes` + 3. Models (`models`) - needs vendors + model types + 4. Business units (`businessunits`) + 5. Location types, then Locations (`locations/types`, `locations`) - the + legacy LocationOnly machines land here, not as assets + 6. Operating systems (`operatingsystems`) + 7. Asset statuses (`assets/statuses`) - from legacy `machinestatus` + 8. Relationship types (`assets/relationshiptypes`) - from legacy + `relationshiptypes` + 9. Notification types (`notifications/types`) + 10. Per-plugin subtypes: computer types (from `pctype`), machine types, + printer types, network device types, measuring-tool types + 11. Support teams + support-team contacts - see the DECIDED disposition in + section 4; import these BEFORE applications because + `applications.supportteamid` points at them + 12. Applications (`applications`) and their versions; import legacy `topics` + as applications too (KB links to applications, section 3) +2. **Assets, per type** (each creates the core Asset row plus its extension): + computers, machines, printers, network devices, measuring tools, and USB + devices. Fan out the legacy `machines` table by category (section 3). +3. **Communications**: the primary IP is set through the asset payload's + `ipaddress` field during step 2. There is no bulk-communications endpoint; + see the mapping note. +4. **Relationships** (`assets/relationships`): needs both endpoint assets and + the relationship types to already exist. +5. **Installed applications**: attach apps to computers + (`computers/{id}/apps`), needs computers + applications. +6. **Knowledge base, notifications, warranties, USB checkouts** (including + backdated history). +7. **Custom fields**: for any legacy column with no home in the target schema, + define a custom field for the asset type and store the value per asset. + +--- + +## 3. Full table-by-table mapping + +Legend: `->` maps to. Endpoints are relative to `http://localhost:5001`. "NK" +is the natural key used for the idempotent lookup (section 5). + +### 3.1 The `machines` hub fans out into the asset plugins + +`machines` (885 rows) is the central legacy asset table. Two columns drive the +fan-out: `machinetypeid` (what the asset physically is) and `pctypeid` (a +computer's sub-type). Route each row by `machinetypeid`: + +| legacy `machinetypeid` | `machinetypes.machinetype` | target plugin | subtype source | +|---|---|---|---| +| 1 | LocationOnly (also `islocationonly=1`) | core **Locations** (NOT an asset) | `locationtype` | +| 33 | PC | **computers** | `computertype` <- `pctype.typename` via `machines.pctypeid` | +| 20 | Server | **computers** | computer type "Server" | +| 15 | Printer | **printers** | `printertype` | +| 16 Access Point / 17 IDF / 18 Camera / 19 Switch / 46 Firewall | | **network** | `networkdevicetype` | +| 44 | USB Device | **usb** | (usb device) | +| 23 Measuring Machine / 3 CMM / 48 Spline Checker / 8 Eddy Current / 47 Inspection | | **measuringtools** (gage-lab judgment call; ADR-005) | `measuringtooltype` | +| 2,4,5,6,7,9,10,11,12,13,14,21,22,24,25,45 (lathes, mills, welders, grinders, ...) | | **machines** | `machinetype` | + +This mapping is a recommended default, not a hard rule; a site may re-route a +`machinetypeid` (for example send CMM to `machines` rather than +`measuringtools`). Decide the routing table once, up front. + +Common `machines` columns -> core Asset fields (same for every target plugin): + +| legacy column | target field | notes | +|---|---|---| +| `machinenumber` | `assetnumber` | the business identifier / NK | +| `alias` or `hostname` | `name` | layperson label | +| `serialnumber` | `serialnumber` | | +| `machinestatusid` | `statusid` | remap via `machinestatus` -> asset statuses | +| `businessunitid` | `businessunitid` | remap via imported business units | +| `mapleft` | `mapx` | | +| `maptop` | `mapy` | | +| `machinenotes` | `notes` | | +| `dateadded` | `createddate` | import mode only | +| `lastupdated` | `modifieddate` | import mode only | + +Per-plugin extension fields: + +- **computers** (`POST /api/computers`): `hostname` <- `machines.hostname`, + `osid` <- remapped `machines.osid`, `computertypeid` <- computer type from + `pctype`, `loggedinuser`, `lastboottime`, `vendorid`, `modelnumberid`, + `ipaddress` <- `machines.ipaddress1` (primary IP). NK: `assetnumber`. +- **machines** (`POST /api/machines`): `machinetypeid`, `vendorid`, + `modelnumberid`, `controllervendorid`/`controllermodelid` (from + `controllertypes` remapped to vendors/models), `requiresmanualconfig` <- + `requires_manual_machine_config`, `islocationonly`. NK: `assetnumber`. +- **printers** (`POST /api/printers`): see 3.2 (authoritative source is the + legacy `printers` table). +- **network** (`POST /api/network`): `networkdevicetypeid`, `hostname`, + `vendorid`, `ipaddress` <- `machines.ipaddress1`. NK: `assetnumber`. +- **measuringtools** (`POST /api/measuringtools`): `measuringtooltypeid`, + calibration fields where known. NK: `assetnumber`. + +### 3.2 Reference and lookup tables + +| legacy table | target endpoint | field mapping | NK | +|---|---|---|---| +| `vendors` | `POST /api/vendors` | `vendor` -> `vendor` | `vendor` | +| `machinetypes` | `POST /api/modeltypes` | `machinetype` -> `modeltype`; set `category` (Equipment/Computer/...) | `modeltype` | +| `models` | `POST /api/models` | `modelnumber`, `vendorid` (remapped), `machinetypeid` -> `modeltypeid`, `notes`, `image` -> `imageurl`, `documentationpath` -> `documentationurl` | `modelnumber` + `vendor` | +| `businessunits` | `POST /api/businessunits` | `businessunit` -> `businessunit` | `businessunit` | +| `operatingsystems` | `POST /api/operatingsystems` | `operatingsystem` -> `osname` | `osname` (+`osversion`) | +| `machinestatus` | `POST /api/assets/statuses` | `machinestatus` -> `status` | `status` | +| `relationshiptypes` | `POST /api/assets/relationshiptypes` | `relationshiptype` -> `relationshiptype`, `description` | `relationshiptype` | +| `notificationtypes` | `POST /api/notifications/types` | `typename`, `typedescription`, `typecolor` | `typename` | +| `pctype` | `POST /api/computers/types` | `typename` -> `computertype`, `description` | `computertype` | +| `subnettypes` | (see subnets) | used as `subnettype` string on subnets | - | +| `subnets` | `POST /api/network/subnets` | `cidr`, `description` -> `name`/`description`, `vlan` -> create VLAN first (`POST /api/network/vlans`) then `vlanid`, `subnettypeid` -> `subnettype` name | `cidr` | +| `dashboarddefaults` | `POST /api/dashboarddefaults` | `ipaddress` -> `ipaddress`, `businessunitid` (remapped), `description` | `ipaddress` | +| `controllertypes` | remap into `vendors` + `models` | e.g. "Fanuc" -> a Vendor; the controller model -> a Model; then set `controllervendorid`/`controllermodelid` on the machine | - | +| `comstypes` | `communicationtypes` (seeded, no API) | ensure `flask seed reference-data` created IP/Serial/USB/... before importing comms | - | + +Note on communication types: the classic `comstypes.typename` values +(IP, Serial, Network_Interface, USB, Parallel, VNC, FTP, DNC) correspond to the +seeded `communicationtypes.comtype`. They are created by the reference-data seed, +not imported per-row. + +### 3.3 Applications, topics, installed apps + +| legacy table | target endpoint | field mapping | NK | +|---|---|---|---| +| `applications` | `POST /api/applications` | `appname`, `appdescription`, `supportteamid` (remapped, section 4), `isinstallable`, `applicationnotes`, `installpath`, `applicationlink`, `documentationpath`, `ishidden`, `isprinter`, `islicenced`, `image` | `appname` | +| `appversions` | `POST /api/applications/{appid}/versions` | `version`, `releasedate`, `notes` | `version` (per app) | +| `topics` | `POST /api/applications` | `topics` is a near-clone of `applications` and `knowledgebase.appid` points at it; import each distinct topic as an Application (`appname` = topic name), so KB links resolve against `applications` | `appname` | +| `installedapps` | `POST /api/computers/{computerid}/apps` | body `{appid, appversionid}`; resolve `machineid` -> the imported computer, `appid`/`appversionid` -> imported app + version | (computerid, appid) | + +`installedapps` only makes sense for computer-class assets; skip rows whose +`machineid` did not map to a computer. + +### 3.4 Communications + +| legacy table | target | field mapping | notes | +|---|---|---|---| +| `communications` (comstypeid=1, isprimary) | asset `ipaddress` on create/update | `address` -> `ipaddress` | Sets the primary IP communication for the asset. | +| `communications` (other comstypeids / secondary rows) | none yet | | No bulk-communication create endpoint exists. Import the primary IP only; capture extra interfaces as custom fields, or defer. | + +### 3.5 Relationships + +| legacy table | target endpoint | field mapping | +|---|---|---| +| `machinerelationships` | `POST /api/assets/relationships` | `machineid` -> `sourceassetid` (the imported asset id), `related_machineid` -> `targetassetid`, `relationshiptypeid` -> `relationshiptypeid` remapped by name, `relationship_notes` -> `notes` | + +Suggested legacy-name -> target relationship-type mapping (create these types +first, or map onto the canonical `partof`/`controls`/`connectedto`): + +| legacy `relationshiptype` | recommended target | +|---|---| +| Controls | Controls | +| Controlled By | Controls (reverse the source/target) | +| Dualpath | Dualpath (or `connectedto` with label "dualpath") | +| Cluster Member | partof | +| Backup For | Backup For | +| Master-Slave | Controls | +| Contains | partof | +| Stored At | Stored At | +| Connected To | connectedto | + +Resolve each machine id to the asset id you got back when you created that +asset (keep a `legacy_machineid -> assetid` map as you import). + +### 3.6 Knowledge base, notifications, warranties, USB + +| legacy table | target endpoint | field mapping | NK | +|---|---|---|---| +| `knowledgebase` | `POST /api/knowledgebase` | `shortdescription`, `linkurl`, `keywords`, `appid` (remapped to the imported application/topic); `lastupdated` -> `modifieddate` in import mode | `linkurl` (fallback `shortdescription`) | +| `notifications` | `POST /api/notifications` | `notification`, `notificationtypeid` (remapped), `businessunitid` (remapped), `starttime`, `endtime`, `ticketnumber`, `link`, `isshopfloor`, `employeesso`; note legacy `endtime` sentinel `2099-00-03 09:52:32` is invalid - drop or clamp it | `ticketnumber` when set, else append-only | +| `warranties` | `POST /api/warranty` | `warrantyname`/`servicelevel` -> `servicelevel`, `enddate` -> `enddate`, link the covered asset via `assetids: [assetid]`; set `vendor` (required) from the source or "Dell"; `servicetag` if known | `servicetag` + `vendor` | +| `usbcheckouts` | see below | historical checkout/checkin events | - | + +USB devices and their history: + +1. Create each USB device (legacy `machines` rows with `machinetypeid=44`, or a + dedicated device list) via `POST /api/usb` in selfhosted mode with body + `{device_id: , device_desc, locker_location}`. NK: `device_id`. +2. Replay each `usbcheckouts` row as a checkout then (if returned) a checkin, + with import-mode backdating: + - `POST /api/usb/{device_id}/checkout` body + `{badge: , reason: , checkouttime: }` + - if `checkin_time` is set: + `POST /api/usb/{device_id}/checkin` body + `{badge: , sanitized: , notes: , checkintime: }` + + The `checkouttime`/`checkintime` overrides are honored only in import mode. + +### 3.7 Anything unmappable -> custom fields + +For a legacy column with no target field (for example `machines.logicmonitorurl`, +`machines.fqdn`, `printers.printerpin`), define a custom field on the asset type +and store the value per asset: + +- `POST /api/customfields` body `{assettypeid, label, datatype}` (once per field) +- `PUT /api/customfields/asset/{assetid}` body `{values: {: }}` + +Custom-field values are not timestamped, so they carry no history. + +--- + +## 4. Tables with no target yet + +These legacy tables have no import target in the current schema. The +dispositions below are DECIDED, not open questions. + +### DECIDED: not migrated + +- **`dncconfig` and `commconfig`** - intentionally NOT migrated. DNC + communication settings drift constantly, so a one-shot import of stale data + has little value. The plan is a future DNC feature fed live by the GE-Enforce + collector/reporting tool rather than a historical import. When that DNC + support is eventually built, the expected ingestion pattern is one attribute + at a time across the whole facility (for example, sweep every machine's baud + rate in one pass, then ports, and so on) via the GE-Enforce collector, so the + future design should favor per-field fleet-wide updates over per-machine + full-record imports. + +### DECIDED: migrated into an upcoming model (do not build here) + +- **`supportteams` and `appowners`** - WILL be migrated, but into a new + `supportteams` / `supportteamcontacts` model that is being built separately + immediately after this task. Legacy `supportteams` (teams, with `teamurl`) + becomes the teams table; legacy `appowners` becomes the contacts, linked to a + team via `supportteams.appownerid`. Import ordering: create teams and their + contacts BEFORE applications, because `applications.supportteamid` references + a team. Do not build these entities as part of the import work - just point + the two legacy tables at that upcoming target. + +### DECIDED: skip (structure only or low value) + +- **`compliance`, `compliancescans`** - 0 rows in `prodscratch`. No data to + migrate; a future compliance plugin would own them. Skip. +- **`ednc_installations`, `ednc_logs`** - 0 rows, and they belong to the eDNC + tooling rather than the asset catalog. Skip. +- **`distributiongroups`** (2 rows) - email distribution lists referenced by + `businessunits.distributiongroupid`. No target; skip, or attach as a business + unit custom field if a site needs it. +- **`functionalaccounts`** (7 rows) - service-account concept referenced by + `pctype`/`machinetypes`; no equivalent in the new schema. Skip, or capture as + a computer-type custom field. +- **`skilllevels`** (2 rows) - orphaned lookup (no FK from `machines`). Skip. + +--- + +## 5. Idempotency recipe and a worked importer + +The endpoints are NOT upserts. The idempotent unit is a two-step recipe that +composes with import mode: + +1. **Look up** the row by its natural key using the exact-match list filter. +2. If found, **PUT** to update it; if not, **POST** to create it. + +Each import-relevant list endpoint has an exact-match filter for its natural key +(added for exactly this purpose): + +| entity | lookup | +|---|---| +| assets (all 5 plugins) | `GET /api/{plugin}?assetnumber=` | +| vendors | `GET /api/vendors?vendor=` | +| models | `GET /api/models?modelnumber=&vendor=` | +| model types | `GET /api/modeltypes?modeltype=` | +| business units | `GET /api/businessunits?businessunit=` | +| locations | `GET /api/locations?locationname=` | +| operating systems | `GET /api/operatingsystems?osname=` | +| applications | `GET /api/applications?appname=` | +| knowledge base | `GET /api/knowledgebase?linkurl=` | +| warranties | `GET /api/warranty?servicetag=&vendor=` | +| notifications | `GET /api/notifications?ticketnumber=` | +| USB devices | `GET /api/usb/{device_id}` (exact by id) | + +### Worked example + +A small, dependency-free importer (`requests`) that logs in, does the +lookup-then-upsert loop in import mode, supports a `--dry-run` flag, and reports +errors without aborting the whole run: + +```python +import argparse +import requests + +BASE = "http://localhost:5001" + + +class ImportClient: + def __init__(self, username, password, dryrun=False): + self.session = requests.Session() + self.dryrun = dryrun + resp = self.session.post( + f"{BASE}/api/auth/login", + json={"username": username, "password": password}, + ) + resp.raise_for_status() + token = resp.json()["data"]["access_token"] + # X-Import-Mode makes createddate/modifieddate passthrough take effect. + self.session.headers.update({ + "Authorization": f"Bearer {token}", + "X-Import-Mode": "true", + }) + + def lookup(self, path, params): + """Return the first matching row, or None.""" + resp = self.session.get(f"{BASE}{path}", params=params) + resp.raise_for_status() + rows = resp.json().get("data") or [] + return rows[0] if rows else None + + def upsert(self, path, idfield, lookupparams, payload): + """Lookup by natural key; PUT if found, else POST. Returns the row.""" + existing = self.lookup(path, lookupparams) + if self.dryrun: + verb = "PUT" if existing else "POST" + print(f"[dry-run] {verb} {path} {lookupparams}") + return existing or payload + if existing: + rowid = existing[idfield] + resp = self.session.put(f"{BASE}{path}/{rowid}", json=payload) + else: + resp = self.session.post(f"{BASE}{path}", json=payload) + if resp.status_code >= 400: + # report and keep going; a single bad row must not abort the run + print(f"ERROR {resp.status_code} {path}: {resp.text[:200]}") + return None + return resp.json()["data"] + + +def import_vendors(client, legacyrows): + for row in legacyrows: + client.upsert( + "/api/vendors", + idfield="vendorid", + lookupparams={"vendor": row["vendor"]}, + payload={ + "vendor": row["vendor"], + # legacy history preserved because X-Import-Mode is set + "createddate": row.get("dateadded"), + "modifieddate": row.get("lastupdated"), + }, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--user", required=True) + parser.add_argument("--password", required=True) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + client = ImportClient(args.user, args.password, dryrun=args.dry_run) + # read legacy rows from prodscratch (read-only) and call the import_* fns + # in the order of section 2, keeping a legacy-id -> new-id map as you go. +``` + +Keep a `legacy_id -> new_id` dictionary for every entity as you import it; you +need it to remap foreign keys (a machine's `businessunitid`, a checkout's +`machineid`, a relationship's `machineid`/`related_machineid`, and so on). + +--- + +## 6. Verification: row-count parity + +After each phase, compare counts. Legacy side (read-only), for example: + +```bash +docker exec dev-mysql mysql -uroot -prootpassword prodscratch \ + -e "SELECT COUNT(*) FROM vendors;" +``` + +New side, via the API pagination metadata (`meta.pagination.total`): + +```bash +curl -s "http://localhost:5001/api/vendors?per_page=1" \ + -H "Authorization: Bearer " | jq '.meta.pagination.total' +``` + +Suggested parity checks: + +| entity | legacy count | new count | +|---|---|---| +| vendors | `SELECT COUNT(*) FROM vendors` | `GET /api/vendors` total | +| models | `SELECT COUNT(*) FROM models` | `GET /api/models` total | +| business units | `SELECT COUNT(*) FROM businessunits` | `GET /api/businessunits` total | +| applications | `SELECT COUNT(*) FROM applications` | `GET /api/applications?showhidden=true` total | +| knowledge base | `SELECT COUNT(*) FROM knowledgebase WHERE isactive` | `GET /api/knowledgebase` total | +| computers | `SELECT COUNT(*) FROM machines WHERE machinetypeid IN (33,20)` | `GET /api/computers` total | +| machines | `SELECT COUNT(*) FROM machines WHERE machinetypeid IN (2,4,5,6,7,9,10,11,12,13,14,21,22,24,25,45)` | `GET /api/machines` total | +| printers | `SELECT COUNT(*) FROM printers WHERE isactive` | `GET /api/printers` total | +| network devices | `SELECT COUNT(*) FROM machines WHERE machinetypeid IN (16,17,18,19,46)` | `GET /api/network` total | +| USB devices | `SELECT COUNT(*) FROM machines WHERE machinetypeid=44` | `GET /api/usb` total | +| relationships | `SELECT COUNT(*) FROM machinerelationships WHERE isactive` | per-asset `GET /api/assets/{id}/relationships` | +| USB checkouts | `SELECT COUNT(*) FROM usbcheckouts` | `GET /api/usb/checkouts` | + +Exact counts will differ where the fan-out routing table (section 3.1) sends a +`machinetypeid` to a different plugin than the example above; adjust the legacy +`WHERE` clause to match the routing you chose. Investigate any gap beyond that. diff --git a/docs/PLUGIN-HOOKS.md b/docs/PLUGIN-HOOKS.md index 995f758..8f90083 100644 --- a/docs/PLUGIN-HOOKS.md +++ b/docs/PLUGIN-HOOKS.md @@ -9,7 +9,7 @@ The contract is locked in [ADR-001](../docs/adr/ADR-001-asset-as-platform-contra The framework declares its contract version in `shopdb/__init__.py`: ```python -__contract_version__ = '0.7.0' +__contract_version__ = '0.8.0' ``` Each plugin's `manifest.json` declares the range of contract versions it supports: @@ -428,7 +428,10 @@ What `shopdb.api` exposes: - Responses: `success_response`, `error_response`, `paginated_response`, `ErrorCodes` - Pagination: `get_pagination_params`, `paginate_query` +- Authorization: `require_permission`, `require_role` - Helpers: `audit_log`, `resolve_asset_position` +- Import mode: `apply_import_timestamps`, `import_mode_active`, + `parse_import_datetime` - Legacy employee directory: `employee_connection` ```python @@ -479,6 +482,30 @@ position = resolve_asset_position(asset) See [ADR-001](../docs/adr/ADR-001-asset-as-platform-contract.md) for the position resolution algorithm. +### Import mode (legacy timestamp passthrough) + +Bulk imports from the classic ASP shopdb need to preserve each row's original +`createddate` / `modifieddate` instead of stamping "now". `apply_import_timestamps` +does this, gated so it never affects normal traffic: it only acts when the +caller is an admin AND sent the `X-Import-Mode: true` request header. + +```python +from shopdb.api import apply_import_timestamps + +asset = Asset(assetnumber=data['assetnumber'], ...) +db.session.add(asset) +# In import mode, stamp legacy createddate/modifieddate from the payload. +# No-op for normal callers, or when the payload omits the fields. +apply_import_timestamps(asset, data) +db.session.commit() +``` + +`import_mode_active()` returns the same admin-plus-header predicate, for guarding +other backdated behavior (for example accepting a historical `checkouttime`). +`parse_import_datetime(value)` parses both ISO `2020-01-05T12:00:00` and legacy +`YYYY-MM-DD HH:MM:SS` into naive UTC. See [docs/IMPORT-API.md](IMPORT-API.md) for +the full migration operator manual. + ## Removed hooks The following hooks existed in early drafts and have been removed for v1: diff --git a/plugins/computers/api/routes.py b/plugins/computers/api/routes.py index 5c5acf5..c4816bb 100644 --- a/plugins/computers/api/routes.py +++ b/plugins/computers/api/routes.py @@ -7,7 +7,7 @@ from shopdb.api import db, Asset, AssetType, OperatingSystem, Application, AppVe from ..models import Computer, ComputerType, ComputerInstalledApp, AccessProtocol, ComputerAccess -from shopdb.api import require_permission, require_role +from shopdb.api import require_permission, require_role, apply_import_timestamps computers_bp = Blueprint('computers', __name__) @@ -323,6 +323,10 @@ def list_computers(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Asset.isactive == True) + # Exact-match natural-key lookup for idempotent import (asset number). + if exactassetnumber := request.args.get('assetnumber'): + query = query.filter(Asset.assetnumber == exactassetnumber) + # Search filter if search := request.args.get('search'): query = query.filter( @@ -551,6 +555,9 @@ def create_computer(): # Remote-access protocols _sync_access_methods(comp, data) + # Preserve legacy timestamps in import mode (no-op otherwise) + apply_import_timestamps(asset, data) + # Audit log AuditLog.log('created', 'Computer', entityid=comp.computerid, entityname=data.get('hostname') or data['assetnumber']) @@ -655,6 +662,7 @@ def update_computer(computer_id: int): AuditLog.log('updated', 'Computer', entityid=comp.computerid, entityname=comp.hostname or asset.assetnumber, changes=changes) + apply_import_timestamps(asset, data) db.session.commit() result = asset.to_dict() diff --git a/plugins/knowledgebase/api/routes.py b/plugins/knowledgebase/api/routes.py index 17110fa..9509917 100644 --- a/plugins/knowledgebase/api/routes.py +++ b/plugins/knowledgebase/api/routes.py @@ -16,7 +16,7 @@ from shopdb.api import ( from ..models import KnowledgeBase -from shopdb.api import require_permission, require_role +from shopdb.api import require_permission, require_role, apply_import_timestamps knowledgebase_bp = Blueprint('knowledgebase', __name__) @@ -42,6 +42,13 @@ def list_articles(): if appid := request.args.get('appid'): query = query.filter(KnowledgeBase.appid == int(appid)) + # Exact-match natural-key lookups for idempotent import. linkurl is the + # stable natural key; shortdescription (the title) is offered as a fallback. + if exactlinkurl := request.args.get('linkurl'): + query = query.filter(KnowledgeBase.linkurl == exactlinkurl) + if exacttitle := request.args.get('shortdescription'): + query = query.filter(KnowledgeBase.shortdescription == exacttitle) + # Sort options sort = request.args.get('sort', 'clicks') order = request.args.get('order', 'desc') @@ -165,6 +172,7 @@ def create_article(): ) db.session.add(article) + apply_import_timestamps(article, data) db.session.commit() return success_response(article.to_dict(), message='Article created', http_code=201) @@ -195,6 +203,7 @@ def update_article(link_id: int): if key in data: setattr(article, key, data[key]) + apply_import_timestamps(article, data) db.session.commit() return success_response(article.to_dict(), message='Article updated') diff --git a/plugins/machines/api/routes.py b/plugins/machines/api/routes.py index 4c70b01..53c192f 100644 --- a/plugins/machines/api/routes.py +++ b/plugins/machines/api/routes.py @@ -7,7 +7,7 @@ from shopdb.api import db, Asset, AssetType, Vendor, Model, AuditLog, success_re from ..models import Machine, MachineType -from shopdb.api import require_permission, require_role +from shopdb.api import require_permission, require_role, apply_import_timestamps machines_bp = Blueprint('machines', __name__) @@ -170,6 +170,10 @@ def list_machines(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Asset.isactive == True) + # Exact-match natural-key lookup for idempotent import (asset number). + if exactassetnumber := request.args.get('assetnumber'): + query = query.filter(Asset.assetnumber == exactassetnumber) + # Search filter if search := request.args.get('search'): query = query.filter( @@ -337,6 +341,9 @@ def create_machine(): db.session.add(mach) db.session.flush() + # Preserve legacy timestamps in import mode (no-op otherwise) + apply_import_timestamps(asset, data) + # Audit log AuditLog.log('created', 'Machine', entityid=mach.machineid, entityname=data['assetnumber']) @@ -412,6 +419,7 @@ def update_machine(machine_id: int): AuditLog.log('updated', 'Machine', entityid=mach.machineid, entityname=asset.assetnumber, changes=changes) + apply_import_timestamps(asset, data) db.session.commit() result = asset.to_dict() diff --git a/plugins/measuringtools/api/routes.py b/plugins/measuringtools/api/routes.py index 8922061..cf2e60d 100644 --- a/plugins/measuringtools/api/routes.py +++ b/plugins/measuringtools/api/routes.py @@ -19,7 +19,7 @@ from shopdb.api import ( db, Asset, AssetType, AuditLog, success_response, error_response, paginated_response, ErrorCodes, get_pagination_params, paginate_query, - require_permission, + require_permission, apply_import_timestamps, ) from ..models import MeasuringTool, MeasuringToolType, derive_status, STATUS_COLORS @@ -165,6 +165,9 @@ def list_tools(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Asset.isactive == True) + # Exact-match natural-key lookup for idempotent import (asset number). + if exactassetnumber := request.args.get('assetnumber'): + query = query.filter(Asset.assetnumber == exactassetnumber) if search := request.args.get('search'): query = query.filter(db.or_( Asset.assetnumber.ilike(f'%{search}%'), @@ -270,6 +273,7 @@ def create_tool(): db.session.add(tool) db.session.flush() + apply_import_timestamps(asset, data) AuditLog.log('created', 'MeasuringTool', entityid=tool.measuringtoolid, entityname=asset.assetnumber) db.session.commit() @@ -321,6 +325,7 @@ def update_tool(tool_id: int): if changes: AuditLog.log('updated', 'MeasuringTool', entityid=tool.measuringtoolid, entityname=asset.assetnumber, changes=changes) + apply_import_timestamps(asset, data) db.session.commit() return success_response(_merged(tool), message='Measuring tool updated') diff --git a/plugins/network/api/routes.py b/plugins/network/api/routes.py index 88c2df4..5f222c0 100644 --- a/plugins/network/api/routes.py +++ b/plugins/network/api/routes.py @@ -7,7 +7,7 @@ from shopdb.api import db, Asset, AssetType, Vendor, AuditLog, success_response, from ..models import NetworkDevice, NetworkDeviceType, Subnet, VLAN -from shopdb.api import require_permission, require_role +from shopdb.api import require_permission, require_role, apply_import_timestamps network_bp = Blueprint('network', __name__) @@ -172,6 +172,10 @@ def list_network_devices(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Asset.isactive == True) + # Exact-match natural-key lookup for idempotent import (asset number). + if exactassetnumber := request.args.get('assetnumber'): + query = query.filter(Asset.assetnumber == exactassetnumber) + # Search filter if search := request.args.get('search'): query = query.filter( @@ -376,6 +380,9 @@ def create_network_device(): db.session.add(netdev) db.session.flush() + # Preserve legacy timestamps in import mode (no-op otherwise) + apply_import_timestamps(asset, data) + # Audit log AuditLog.log('created', 'NetworkDevice', entityid=netdev.networkdeviceid, entityname=data.get('hostname') or data['assetnumber']) @@ -458,6 +465,7 @@ def update_network_device(device_id: int): AuditLog.log('updated', 'NetworkDevice', entityid=netdev.networkdeviceid, entityname=netdev.hostname or asset.assetnumber, changes=changes) + apply_import_timestamps(asset, data) db.session.commit() result = asset.to_dict() diff --git a/plugins/notifications/api/routes.py b/plugins/notifications/api/routes.py index 3eea00d..1274906 100644 --- a/plugins/notifications/api/routes.py +++ b/plugins/notifications/api/routes.py @@ -288,6 +288,11 @@ def list_notifications(): if type_id := request.args.get('typeid', request.args.get('type_id')): query = query.filter(Notification.notificationtypeid == int(type_id)) + # Exact-match lookup for idempotent import. Notifications have no strong + # natural key; ticketnumber is the best available when a ticket is set. + if exactticket := request.args.get('ticketnumber'): + query = query.filter(Notification.ticketnumber == exactticket) + # Current filter (active based on dates) if request.args.get('current', 'false').lower() == 'true': now = datetime.now(timezone.utc).replace(tzinfo=None) diff --git a/plugins/printers/api/asset_routes.py b/plugins/printers/api/asset_routes.py index a4eaf1d..13b5216 100644 --- a/plugins/printers/api/asset_routes.py +++ b/plugins/printers/api/asset_routes.py @@ -19,7 +19,7 @@ from ..services import ( logger = logging.getLogger(__name__) -from shopdb.api import require_permission, require_role +from shopdb.api import require_permission, require_role, apply_import_timestamps printers_asset_bp = Blueprint('printers_asset', __name__) @@ -235,6 +235,10 @@ def list_printers(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Asset.isactive == True) + # Exact-match natural-key lookup for idempotent import (asset number). + if exactassetnumber := request.args.get('assetnumber'): + query = query.filter(Asset.assetnumber == exactassetnumber) + # Search filter if search := request.args.get('search'): query = query.filter( @@ -538,6 +542,9 @@ def create_printer(): ) db.session.add(comm) + # Preserve legacy timestamps in import mode (no-op otherwise) + apply_import_timestamps(asset, data) + db.session.commit() result = asset.to_dict() @@ -613,6 +620,7 @@ def update_printer(printer_id: int): elif comm: comm.ipaddress = None + apply_import_timestamps(asset, data) db.session.commit() result = asset.to_dict() diff --git a/plugins/usb/api/selfhosted.py b/plugins/usb/api/selfhosted.py index 4009e2f..77d2aeb 100644 --- a/plugins/usb/api/selfhosted.py +++ b/plugins/usb/api/selfhosted.py @@ -21,6 +21,7 @@ from datetime import datetime, timezone from shopdb.api import ( db, success_response, error_response, ErrorCodes, get_pagination_params, paginated_response, + apply_import_timestamps, import_mode_active, parse_import_datetime, ) from ..models import USBDevice, USBCheckout @@ -147,6 +148,7 @@ def create_device(data): storagelocation=data.get('locker_location'), ischeckedout=False, isactive=True) db.session.add(device) + apply_import_timestamps(device, data) db.session.commit() return success_response(_device_dict(device), message='Device created', http_code=201) @@ -189,13 +191,20 @@ def checkout_device(device_id, data): return error_response(ErrorCodes.CONFLICT, 'Device is already checked out', http_code=409) name = _resolve_name(badge) now = datetime.now(timezone.utc).replace(tzinfo=None) + # Backdated import: an admin import request may pass the historical + # checkouttime so migrated usbcheckouts rows keep their real event time. + eventtime = now + if import_mode_active(): + override = parse_import_datetime(data.get('checkouttime')) + if override is not None: + eventtime = override db.session.add(USBCheckout(usbdeviceid=device.usbdeviceid, machineid=0, sso=badge, - checkoutname=name, checkouttime=now, + checkoutname=name, checkouttime=eventtime, checkoutreason=data.get('reason'))) device.ischeckedout = True device.currentuserid = badge device.currentusername = name - device.currentcheckoutdate = now + device.currentcheckoutdate = eventtime if data.get('locker_location'): device.storagelocation = data['locker_location'] db.session.commit() @@ -215,7 +224,14 @@ def checkin_device(device_id, data): .filter_by(usbdeviceid=device.usbdeviceid, checkintime=None) .order_by(USBCheckout.checkouttime.desc()).first()) if open_checkout: - open_checkout.checkintime = datetime.now(timezone.utc).replace(tzinfo=None) + checkintime = datetime.now(timezone.utc).replace(tzinfo=None) + # Backdated import: accept the historical checkintime from an admin + # import request so returned checkouts keep their real return time. + if import_mode_active(): + override = parse_import_datetime(data.get('checkintime')) + if override is not None: + checkintime = override + open_checkout.checkintime = checkintime open_checkout.waswiped = bool(data.get('sanitized')) open_checkout.checkinnotes = data.get('notes') device.ischeckedout = False diff --git a/plugins/warranty/api/routes.py b/plugins/warranty/api/routes.py index dab775e..41ddb5d 100644 --- a/plugins/warranty/api/routes.py +++ b/plugins/warranty/api/routes.py @@ -77,6 +77,11 @@ def list_warranties(): query = Warranty.query if request.args.get('active', 'true').lower() != 'false': query = query.filter_by(isactive=True) + # Exact-match natural-key lookup for idempotent import (servicetag + vendor). + if exactservicetag := request.args.get('servicetag'): + query = query.filter(Warranty.servicetag == exactservicetag) + if exactvendor := request.args.get('vendor'): + query = query.filter(Warranty.vendor == exactvendor) assetid = request.args.get('assetid', type=int) if assetid: query = (query.join(WarrantyAsset, WarrantyAsset.warrantyid == Warranty.warrantyid) diff --git a/shopdb/__init__.py b/shopdb/__init__.py index 023c8a3..834b15f 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -23,7 +23,7 @@ from .plugins import plugin_manager # 0.7.0: added the four ADR-010 frontend-contribution hooks (get_settings_cards, # get_asset_panels, get_map_overlays, get_asset_presentation), consumed by the # GET /api/pluginui/* endpoints. Four additive optional hooks, one minor bump. -__contract_version__ = '0.7.0' +__contract_version__ = '0.8.0' # Product release version (see ADR-007). The product version and the # plugin-contract version above are distinct series with independent diff --git a/shopdb/api/__init__.py b/shopdb/api/__init__.py index 252cb4d..ccae455 100644 --- a/shopdb/api/__init__.py +++ b/shopdb/api/__init__.py @@ -58,6 +58,13 @@ from shopdb.utils.pagination import get_pagination_params, paginate_query # Authorization decorators for gating plugin write routes from shopdb.utils.authz import require_permission, require_role +# Import-mode helpers: preserve legacy timestamps during a bulk data import +from shopdb.utils.import_mode import ( + apply_import_timestamps, + import_mode_active, + parse_import_datetime, +) + # Legacy employee directory lookup (read-only) used by notifications from shopdb.utils.employee_db import employee_connection @@ -239,6 +246,10 @@ __all__ = [ # Authorization decorators 'require_permission', 'require_role', + # Import-mode helpers + 'apply_import_timestamps', + 'import_mode_active', + 'parse_import_datetime', # Legacy employee directory 'employee_connection', # CMMC USB check-in/out database diff --git a/shopdb/core/api/applications.py b/shopdb/core/api/applications.py index 95dfde4..e411896 100644 --- a/shopdb/core/api/applications.py +++ b/shopdb/core/api/applications.py @@ -14,6 +14,7 @@ from shopdb.utils.responses import ( ErrorCodes ) from shopdb.utils.pagination import get_pagination_params, paginate_query +from shopdb.utils.import_mode import apply_import_timestamps def _computer_models(): @@ -78,6 +79,10 @@ def list_applications(): installable = request.args.get('installable').lower() == 'true' query = query.filter(Application.isinstallable == installable) + # Exact-match natural-key lookup for idempotent import (app name). + if exactappname := request.args.get('appname'): + query = query.filter(Application.appname == exactappname) + if search := request.args.get('search'): query = query.filter( db.or_( @@ -174,6 +179,7 @@ def create_application(): ) db.session.add(app) + apply_import_timestamps(app, data) db.session.flush() AuditLog.log('created', 'Application', entityid=app.appid, entityname=app.appname) @@ -224,6 +230,7 @@ def update_application(app_id: int): AuditLog.log('updated', 'Application', entityid=app.appid, entityname=app.appname, changes=changes) + apply_import_timestamps(app, data) db.session.commit() return success_response(app.to_dict(), message='Application updated') diff --git a/shopdb/core/api/assets.py b/shopdb/core/api/assets.py index f370289..a5915ea 100644 --- a/shopdb/core/api/assets.py +++ b/shopdb/core/api/assets.py @@ -15,6 +15,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps assets_bp = Blueprint('assets', __name__) @@ -470,6 +471,7 @@ def create_asset(): ) db.session.add(asset) + apply_import_timestamps(asset, data) db.session.commit() return success_response(asset.to_dict(), message='Asset created', http_code=201) @@ -512,6 +514,7 @@ def update_asset(asset_id: int): if key in data: setattr(asset, key, data[key]) + apply_import_timestamps(asset, data) db.session.commit() return success_response(asset.to_dict(), message='Asset updated') @@ -657,6 +660,7 @@ def create_asset_relationship(): ) db.session.add(rel) + apply_import_timestamps(rel, data) db.session.commit() return success_response(rel.to_dict(), message='Relationship created', http_code=201) diff --git a/shopdb/core/api/businessunits.py b/shopdb/core/api/businessunits.py index 3b572d3..a72515d 100644 --- a/shopdb/core/api/businessunits.py +++ b/shopdb/core/api/businessunits.py @@ -14,6 +14,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps businessunits_bp = Blueprint('businessunits', __name__) @@ -29,6 +30,10 @@ def list_businessunits(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(BusinessUnit.isactive == True) + # Exact-match natural-key lookup for idempotent import (unit name). + if exactunit := request.args.get('businessunit'): + query = query.filter(BusinessUnit.businessunit == exactunit) + if search := request.args.get('search'): query = query.filter( db.or_( @@ -90,6 +95,7 @@ def create_businessunit(): ) db.session.add(bu) + apply_import_timestamps(bu, data) db.session.commit() return success_response(bu.to_dict(), message='Business unit created', http_code=201) @@ -125,6 +131,7 @@ def update_businessunit(bu_id: int): if key in data: setattr(bu, key, data[key]) + apply_import_timestamps(bu, data) db.session.commit() return success_response(bu.to_dict(), message='Business unit updated') diff --git a/shopdb/core/api/locations.py b/shopdb/core/api/locations.py index 9bae915..460c2dc 100644 --- a/shopdb/core/api/locations.py +++ b/shopdb/core/api/locations.py @@ -14,6 +14,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps locations_bp = Blueprint('locations', __name__) @@ -110,6 +111,10 @@ def list_locations(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Location.isactive == True) + # Exact-match natural-key lookup for idempotent import (location name). + if exactname := request.args.get('locationname'): + query = query.filter(Location.locationname == exactname) + if search := request.args.get('search'): query = query.filter( db.or_( @@ -173,6 +178,7 @@ def create_location(): ) db.session.add(loc) + apply_import_timestamps(loc, data) db.session.commit() return success_response(loc.to_dict(), message='Location created', http_code=201) @@ -210,6 +216,7 @@ def update_location(location_id: int): if key in data: setattr(loc, key, data[key]) + apply_import_timestamps(loc, data) db.session.commit() return success_response(loc.to_dict(), message='Location updated') diff --git a/shopdb/core/api/models.py b/shopdb/core/api/models.py index 1a9d1a0..cd398f6 100644 --- a/shopdb/core/api/models.py +++ b/shopdb/core/api/models.py @@ -14,6 +14,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps models_bp = Blueprint('models', __name__) @@ -35,6 +36,11 @@ def list_models(): if modeltype_id := request.args.get('modeltype', type=int): query = query.filter(Model.modeltypeid == modeltype_id) + # Exact-match natural-key lookup for idempotent import. Natural key is + # modelnumber + vendor; pair this with ?vendor= to disambiguate. + if exactmodelnumber := request.args.get('modelnumber'): + query = query.filter(Model.modelnumber == exactmodelnumber) + if search := request.args.get('search'): query = query.filter(Model.modelnumber.ilike(f'%{search}%')) @@ -105,6 +111,7 @@ def create_model(): ) db.session.add(m) + apply_import_timestamps(m, data) db.session.commit() return success_response(m.to_dict(), message='Model created', http_code=201) @@ -132,6 +139,7 @@ def update_model(model_id: int): if key in data: setattr(m, key, data[key]) + apply_import_timestamps(m, data) db.session.commit() return success_response(m.to_dict(), message='Model updated') diff --git a/shopdb/core/api/modeltypes.py b/shopdb/core/api/modeltypes.py index 1c4cbb1..052f71c 100644 --- a/shopdb/core/api/modeltypes.py +++ b/shopdb/core/api/modeltypes.py @@ -18,6 +18,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps modeltypes_bp = Blueprint('modeltypes', __name__) @@ -36,6 +37,10 @@ def list_modeltypes(): if category := request.args.get('category'): query = query.filter(ModelType.category == category) + # Exact-match natural-key lookup for idempotent import (type name). + if exactmodeltype := request.args.get('modeltype'): + query = query.filter(ModelType.modeltype == exactmodeltype) + if search := request.args.get('search'): query = query.filter(ModelType.modeltype.ilike(f'%{search}%')) @@ -88,6 +93,7 @@ def create_modeltype(): ) db.session.add(mt) + apply_import_timestamps(mt, data) db.session.commit() return success_response(mt.to_dict(), message='Model type created', http_code=201) @@ -124,6 +130,7 @@ def update_modeltype(type_id: int): if key in data: setattr(mt, key, data[key]) + apply_import_timestamps(mt, data) db.session.commit() return success_response(mt.to_dict(), message='Model type updated') diff --git a/shopdb/core/api/operatingsystems.py b/shopdb/core/api/operatingsystems.py index 3960182..216ac84 100644 --- a/shopdb/core/api/operatingsystems.py +++ b/shopdb/core/api/operatingsystems.py @@ -14,6 +14,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps operatingsystems_bp = Blueprint('operatingsystems', __name__) @@ -29,6 +30,13 @@ def list_operatingsystems(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(OperatingSystem.isactive == True) + # Exact-match natural-key lookup for idempotent import. Legacy OS rows have + # only a name; pair with ?osversion= when versions are tracked separately. + if exactosname := request.args.get('osname'): + query = query.filter(OperatingSystem.osname == exactosname) + if exactosversion := request.args.get('osversion'): + query = query.filter(OperatingSystem.osversion == exactosversion) + if search := request.args.get('search'): query = query.filter(OperatingSystem.osname.ilike(f'%{search}%')) @@ -85,6 +93,7 @@ def create_operatingsystem(): ) db.session.add(os) + apply_import_timestamps(os, data) db.session.commit() return success_response(os.to_dict(), message='Operating system created', http_code=201) @@ -112,6 +121,7 @@ def update_operatingsystem(os_id: int): if key in data: setattr(os, key, data[key]) + apply_import_timestamps(os, data) db.session.commit() return success_response(os.to_dict(), message='Operating system updated') diff --git a/shopdb/core/api/vendors.py b/shopdb/core/api/vendors.py index c4cb426..23c84b6 100644 --- a/shopdb/core/api/vendors.py +++ b/shopdb/core/api/vendors.py @@ -14,6 +14,7 @@ from shopdb.utils.responses import ( from shopdb.utils.pagination import get_pagination_params, paginate_query from shopdb.utils.authz import require_permission, require_role +from shopdb.utils.import_mode import apply_import_timestamps vendors_bp = Blueprint('vendors', __name__) @@ -29,6 +30,10 @@ def list_vendors(): if request.args.get('active', 'true').lower() != 'false': query = query.filter(Vendor.isactive == True) + # Exact-match natural-key lookup for idempotent import (vendor name). + if exactvendor := request.args.get('vendor'): + query = query.filter(Vendor.vendor == exactvendor) + if search := request.args.get('search'): query = query.filter(Vendor.vendor.ilike(f'%{search}%')) @@ -83,6 +88,7 @@ def create_vendor(): ) db.session.add(v) + apply_import_timestamps(v, data) db.session.commit() return success_response(v.to_dict(), message='Vendor created', http_code=201) @@ -118,6 +124,7 @@ def update_vendor(vendor_id: int): if key in data: setattr(v, key, data[key]) + apply_import_timestamps(v, data) db.session.commit() return success_response(v.to_dict(), message='Vendor updated') diff --git a/shopdb/utils/import_mode.py b/shopdb/utils/import_mode.py new file mode 100644 index 0000000..6247d71 --- /dev/null +++ b/shopdb/utils/import_mode.py @@ -0,0 +1,104 @@ +"""Import-mode helpers: let admins replay legacy history through the API. + +A migration script (or an LLM driving one) importing the classic ASP shopdb +needs two things the normal API withholds: + + 1. Preserve each row's original createddate/modifieddate instead of stamping + "now" on insert/update. + 2. Backdate event history such as USB checkouts to when they really happened. + +Both are gated. The caller must BOTH be an admin AND send the request header +`X-Import-Mode: true`. Outside import mode every helper here is a no-op, so +wiring a call into a normal create/update path never changes behavior for +regular users. See docs/IMPORT-API.md for the operator manual. +""" + +from datetime import datetime, timezone + +from flask import request +from flask_jwt_extended import verify_jwt_in_request, current_user + + +# Request header a migration client sets to opt a request into import mode. +IMPORT_MODE_HEADER = 'X-Import-Mode' + +# strptime formats accepted for import timestamps, tried in order. Covers the +# ISO 'T' form and the legacy MySQL 'space' form, with and without fractions, +# plus a bare date. +_IMPORT_DATETIME_FORMATS = ( + '%Y-%m-%dT%H:%M:%S.%f', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M', + '%Y-%m-%d %H:%M:%S.%f', + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%d %H:%M', + '%Y-%m-%d', +) + + +def import_mode_active(): + """True when the request is an admin-authenticated import request. + + Needs header `X-Import-Mode: true` AND an admin caller. Safe to call from + any request context: verify_jwt_in_request is optional here so a missing or + bad token yields False instead of raising, and it is idempotent when the + route already ran @jwt_required.""" + header = (request.headers.get(IMPORT_MODE_HEADER) or '').strip().lower() + if header != 'true': + return False + verify_jwt_in_request(optional=True) + user = current_user + return bool(user is not None and user.hasrole('admin')) + + +def parse_import_datetime(value): + """Parse an import timestamp into naive UTC, or None. + + Accepts ISO '2020-01-05T12:00:00', legacy 'YYYY-MM-DD HH:MM:SS', a bare + date, or an already-parsed datetime. A trailing 'Z' is treated as UTC. + Timezone-aware input is converted to UTC then stripped to naive, matching + the repo convention of storing naive-UTC in DB DateTime columns.""" + if value is None: + return None + if isinstance(value, datetime): + parsed = value + else: + text = str(value).strip() + if not text: + return None + if text[-1] in ('Z', 'z'): + text = text[:-1] + parsed = None + for fmt in _IMPORT_DATETIME_FORMATS: + try: + parsed = datetime.strptime(text, fmt) + break + except ValueError: + continue + if parsed is None: + # last resort: let fromisoformat try (handles offsets like +00:00) + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) + return parsed + + +def apply_import_timestamps(instance, data): + """Stamp createddate/modifieddate on a row from the payload, in import mode. + + Call this in a create/update path after building the instance and before + commit. It is a no-op unless ALL hold: import mode is active (admin + + header), the payload carries the field, and the model actually has the + column. Explicitly setting modifieddate also suppresses the column's + onupdate=now default on updates, so legacy history survives edits too.""" + if not data or not import_mode_active(): + return + created = parse_import_datetime(data.get('createddate')) + if created is not None and hasattr(instance, 'createddate'): + instance.createddate = created + modified = parse_import_datetime(data.get('modifieddate')) + if modified is not None and hasattr(instance, 'modifieddate'): + instance.modifieddate = modified diff --git a/tests/test_core/test_import_mode.py b/tests/test_core/test_import_mode.py new file mode 100644 index 0000000..6d7011b --- /dev/null +++ b/tests/test_core/test_import_mode.py @@ -0,0 +1,209 @@ +"""Import-mode behavior: admin-gated legacy timestamp + event backdating. + +Covers deliverables of the import-API work: +- createddate/modifieddate passthrough on a reference entity (vendor) and an + asset plugin (computer), gated on admin + X-Import-Mode header. +- both negative cases: no header, and header sent by a non-admin. +- a natural-key exact-match lookup filter added for idempotent import. +- backdated USB checkout time in selfhosted mode. +""" + +import pytest +from werkzeug.security import generate_password_hash + +# Import the usb models at collection time so their tables register on the +# metadata before the per-test db fixture calls create_all. The usb plugin is +# disabled in the dev registry, so its tables would otherwise never be created. +from plugins.usb.models import USBDevice, USBCheckout # noqa: F401 + + +IMPORT_HEADER = {'X-Import-Mode': 'true'} +LEGACY_CREATED = '2020-01-05 08:30:00' +LEGACY_MODIFIED = '2021-06-07T14:15:16' + + +def _headers(auth_headers, importmode=True): + merged = dict(auth_headers) + if importmode: + merged.update(IMPORT_HEADER) + return merged + + +@pytest.fixture +def asset_types(db): + """Seed the asset types the plugin create endpoints look up by name.""" + from shopdb.core.models import AssetType + + for name in ('computer',): + db.session.add(AssetType(assettype=name, pluginname=name, + tablename=name, description=name)) + db.session.commit() + + +@pytest.fixture +def creator_headers(client, db): + """A NON-admin user granted only computers.create/edit, logged in. + + Proves import mode requires admin: this user carries the write permission + but must NOT get timestamp passthrough. + """ + from shopdb.core.models import User, Role, Permission + + perms = [] + for name in ('computers.create', 'computers.edit'): + p = Permission(name=name, description=name, category='computers') + db.session.add(p) + perms.append(p) + role = Role(rolename='pcoperator', description='PC operator') + role.permissions.extend(perms) + db.session.add(role) + db.session.flush() + + user = User(username='pcop', email='pcop@test.local', + passwordhash=generate_password_hash('testpass')) + user.roles.append(role) + db.session.add(user) + db.session.commit() + + response = client.post('/api/auth/login', + json={'username': 'pcop', 'password': 'testpass'}) + assert response.status_code == 200, response.get_json() + token = response.get_json()['data']['access_token'] + return {'Authorization': f'Bearer {token}'} + + +# --------------------------------------------------------------------------- +# Timestamp passthrough (positive) +# --------------------------------------------------------------------------- + +def test_admin_import_mode_preserves_vendor_timestamps(client, db, auth_headers): + """Admin + header: vendor keeps the legacy createddate/modifieddate.""" + payload = {'vendor': 'LegacyVendor', + 'createddate': LEGACY_CREATED, + 'modifieddate': LEGACY_MODIFIED} + response = client.post('/api/vendors', json=payload, + headers=_headers(auth_headers)) + assert response.status_code == 201, response.get_json() + + from shopdb.core.models import Vendor + v = Vendor.query.filter_by(vendor='LegacyVendor').first() + assert v.createddate.year == 2020 and v.createddate.month == 1 + assert v.createddate.day == 5 and v.createddate.hour == 8 + assert v.modifieddate.year == 2021 and v.modifieddate.month == 6 + + +def test_admin_import_mode_preserves_computer_asset_timestamps( + client, db, auth_headers, asset_types): + """Admin + header: the created asset row keeps its legacy createddate.""" + payload = {'assetnumber': 'PC-IMPORT-1', + 'hostname': 'legacy-pc', + 'createddate': LEGACY_CREATED, + 'modifieddate': LEGACY_MODIFIED} + response = client.post('/api/computers', json=payload, + headers=_headers(auth_headers)) + assert response.status_code == 201, response.get_json() + + from shopdb.core.models import Asset + asset = Asset.query.filter_by(assetnumber='PC-IMPORT-1').first() + assert asset.createddate.year == 2020 + assert asset.modifieddate.year == 2021 + + +# --------------------------------------------------------------------------- +# Timestamp passthrough (negative): ignored without header, or for non-admin +# --------------------------------------------------------------------------- + +def test_no_header_ignores_timestamps(client, db, auth_headers): + """Admin but NO X-Import-Mode header: createddate falls back to now.""" + payload = {'vendor': 'NoHeaderVendor', 'createddate': LEGACY_CREATED} + response = client.post('/api/vendors', json=payload, + headers=_headers(auth_headers, importmode=False)) + assert response.status_code == 201, response.get_json() + + from shopdb.core.models import Vendor + v = Vendor.query.filter_by(vendor='NoHeaderVendor').first() + assert v.createddate.year >= 2024 # server-stamped now, not the 2020 payload + + +def test_non_admin_ignores_timestamps(client, db, creator_headers, asset_types): + """Non-admin WITH the write permission + header: timestamps still ignored.""" + payload = {'assetnumber': 'PC-NONADMIN-1', + 'createddate': LEGACY_CREATED} + response = client.post('/api/computers', json=payload, + headers=_headers(creator_headers)) + assert response.status_code == 201, response.get_json() + + from shopdb.core.models import Asset + asset = Asset.query.filter_by(assetnumber='PC-NONADMIN-1').first() + assert asset.createddate.year >= 2024 # not backdated: caller is not admin + + +# --------------------------------------------------------------------------- +# Natural-key lookup filter (idempotency recipe support) +# --------------------------------------------------------------------------- + +def test_vendor_exact_lookup_filter(client, db, auth_headers): + """GET /api/vendors?vendor= exact-matches one vendor for lookup.""" + for name in ('Acme', 'AcmeTools', 'Beta'): + resp = client.post('/api/vendors', json={'vendor': name}, + headers=auth_headers) + assert resp.status_code == 201, resp.get_json() + + listed = client.get('/api/vendors?vendor=Acme', headers=auth_headers) + assert listed.status_code == 200 + rows = listed.get_json()['data'] + assert len(rows) == 1 + assert rows[0]['vendor'] == 'Acme' + + +# --------------------------------------------------------------------------- +# Backdated USB checkout (selfhosted mode) +# +# The usb plugin is disabled in the dev registry, so its blueprint may not be +# registered. Exercise the selfhosted layer directly inside a request context +# carrying the admin JWT + import header, which is what the route delegates to. +# --------------------------------------------------------------------------- + +def _seed_usb_device(db, serialnumber): + from plugins.usb.models import USBDevice + device = USBDevice(serialnumber=serialnumber, ischeckedout=False, isactive=True) + db.session.add(device) + db.session.commit() + return device + + +def test_usb_backdated_checkout(app, db, auth_headers): + """Admin + header: a USB checkout records the historical checkouttime.""" + from plugins.usb.api import selfhosted + from plugins.usb.models import USBDevice, USBCheckout + + device = _seed_usb_device(db, 'USB-IMPORT-1') + headers = _headers(auth_headers) + with app.test_request_context('/api/usb/USB-IMPORT-1/checkout', + method='POST', headers=headers): + resp = selfhosted.checkout_device( + 'USB-IMPORT-1', + {'badge': '570005354', 'checkouttime': '2020-03-04 09:10:11'}) + assert resp.status_code == 201, resp.get_json() + + row = USBCheckout.query.filter_by(usbdeviceid=device.usbdeviceid).first() + assert row.checkouttime.year == 2020 and row.checkouttime.month == 3 + reloaded = db.session.get(USBDevice, device.usbdeviceid) + assert reloaded.currentcheckoutdate.year == 2020 + + +def test_usb_checkout_ignores_backdate_without_header(app, db, auth_headers): + """No import header: the checkouttime override is ignored, uses now.""" + from plugins.usb.api import selfhosted + from plugins.usb.models import USBDevice, USBCheckout + + device = _seed_usb_device(db, 'USB-IMPORT-2') + with app.test_request_context('/api/usb/USB-IMPORT-2/checkout', + method='POST', headers=auth_headers): + resp = selfhosted.checkout_device( + 'USB-IMPORT-2', + {'badge': '570005354', 'checkouttime': '2020-03-04 09:10:11'}) + assert resp.status_code == 201, resp.get_json() + + row = USBCheckout.query.filter_by(usbdeviceid=device.usbdeviceid).first() + assert row.checkouttime.year >= 2024 # server now, not the 2020 override