Add reversible Maximo/WINC machine-adoption import
Machine data arrives as a Maximo asset export (validated against the WINC site) that must update existing machines in place, not create canonical per-table rows, so it does not fit the ShopDB-native/Azure/Thor CSV adapter path. Bulk edits to live machines also need an escape hatch, so the whole commit is journaled and can be undone as one batch. Add a dedicated dry-run analyzer, approval packet, resolver/suggest helpers, and a reversible writer/undo backed by a new importbatches/importjournalentries schema (with migration). Expose preview/commit/batches/undo settings routes and a matching frontend source card and panel, and surface the batch/journal contract through shopdb.api for plugins.
This commit is contained in:
benjamin.baker
278
scripts/site_imports/winc/writer.py
Normal file
278
scripts/site_imports/winc/writer.py
Normal file
@@ -0,0 +1,278 @@
|
||||
"""WINC machine adoption - WRITER (applies the approved plan, reversibly).
|
||||
|
||||
This is the write half of the WINC import; scripts/site_imports/winc/dry_run.py
|
||||
is the read-only half. The writer never invents its own view of the data: it
|
||||
rebuilds the exact same plan the dry-run produced, verifies that plan against the
|
||||
approval packet the human signed off on, and only then writes - recording every
|
||||
change to the shared import journal so the whole batch can be undone.
|
||||
|
||||
The contract, in order (the one documented in approval.py, made concrete here):
|
||||
|
||||
1. rebuild the plan from the same export + live DB,
|
||||
2. build a packet from it,
|
||||
3. compare planhash to the approved packet - abort on mismatch (drift),
|
||||
4. refuse if the packet is blocking (any unresolved value),
|
||||
5. require explicit acknowledgment for every needs-create group,
|
||||
6. write inside ONE transaction, stamping the batch id onto the journal:
|
||||
a. create each acknowledged needs-create business unit (journal insert),
|
||||
b. update each in-scope asset's name / location / business unit
|
||||
(journal update, capturing the pre-image first).
|
||||
|
||||
Everything commits together or not at all. Ambiguous rows (one workstation key
|
||||
matching several machines) are never written - the dry-run flags them and the
|
||||
writer skips them, because guessing which machine an export row means is exactly
|
||||
the kind of cleverness this import refuses.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
|
||||
os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
|
||||
|
||||
from scripts.site_imports import journal
|
||||
from scripts.site_imports.approval import build_packet, packet_matches_plan
|
||||
from scripts.site_imports.resolver import NEEDS_CREATE, RESOLVED, norm
|
||||
from scripts.site_imports.winc.dry_run import (
|
||||
analyze,
|
||||
build_app_connection,
|
||||
)
|
||||
|
||||
# The packet groups the WINC dry-run declares, reused verbatim so the writer
|
||||
# hashes and reads the plan exactly the way approval did.
|
||||
_PACKET_GROUPS = [
|
||||
('businessunit', 'businessunit_status', 'raw_eq1', 'businessunit_name'),
|
||||
('location', 'location_status', 'raw_location', 'location_name'),
|
||||
]
|
||||
|
||||
# References into a created row's id, so undo can tell whether an inserted
|
||||
# business unit is still pointed at by something outside this batch before it
|
||||
# hard-deletes it. Shared with the undo CLI and the web undo path.
|
||||
_UNDO_REFERENCES = {
|
||||
'businessunits': [('assets', 'businessunitid')],
|
||||
}
|
||||
|
||||
|
||||
# References into a created row's id, so undo can tell whether an inserted
|
||||
# business unit is still pointed at by something outside this batch before it
|
||||
# hard-deletes it.
|
||||
# Max width of assets.name; the export description can exceed it (a few rows
|
||||
# carry a whole safety notice in the field), so a set name is clamped rather
|
||||
# than letting the whole transaction fail on one over-long row. The pre-image is
|
||||
# journalled, so the clamp is reversible like any other change.
|
||||
_ASSET_NAME_MAXLEN = 100
|
||||
|
||||
|
||||
class WriterError(Exception):
|
||||
"""The plan cannot be written as approved (drift, blocking, or missing
|
||||
acknowledgment). Carries a human-readable reason; never a partial write."""
|
||||
|
||||
|
||||
def _load_asset_ids(connection):
|
||||
"""assetnumber -> assetid for every asset, so a plan row (which carries the
|
||||
matched asset *number*) can be turned into the id the update targets."""
|
||||
from sqlalchemy import text
|
||||
|
||||
out = {}
|
||||
for assetid, assetnumber in connection.execute(text(
|
||||
'SELECT assetid, assetnumber FROM assets')):
|
||||
out[assetnumber] = assetid
|
||||
return out
|
||||
|
||||
|
||||
def _create_businessunit(connection, batchid, sequence, name):
|
||||
"""Insert one business unit and journal it. Returns (new_id, next_sequence).
|
||||
|
||||
Only ever called for an acknowledged needs-create name, so the journal's
|
||||
insert entry describes strictly a row this batch created."""
|
||||
from sqlalchemy import text
|
||||
|
||||
now = journal._utcnow()
|
||||
result = connection.execute(text(
|
||||
'INSERT INTO businessunits '
|
||||
'(businessunit, createddate, modifieddate, isactive) '
|
||||
'VALUES (:name, :now, :now, 1)'),
|
||||
{'name': name, 'now': now})
|
||||
new_id = result.lastrowid
|
||||
journal.record_insert(connection, batchid, sequence, 'businessunits',
|
||||
new_id)
|
||||
return new_id, sequence + 1
|
||||
|
||||
|
||||
def _update_cell(connection, batchid, sequence, table, targetid, column,
|
||||
oldvalue, newvalue):
|
||||
"""Apply one cell update and journal its pre/post image. Returns the next
|
||||
sequence. Skips (and does not journal) a no-op where the value is unchanged."""
|
||||
from sqlalchemy import text
|
||||
|
||||
if journal._as_text(oldvalue) == journal._as_text(newvalue):
|
||||
return sequence
|
||||
connection.execute(text(
|
||||
'UPDATE %s SET %s = :newvalue WHERE %s = :targetid'
|
||||
% (table, column, _asset_pk(table))),
|
||||
{'newvalue': newvalue, 'targetid': targetid})
|
||||
journal.record_update(connection, batchid, sequence, table, targetid,
|
||||
column, oldvalue, newvalue)
|
||||
return sequence + 1
|
||||
|
||||
|
||||
def _asset_pk(table):
|
||||
# The writer only updates the assets table; keep the mapping explicit rather
|
||||
# than reflecting, so an unexpected table name fails loudly.
|
||||
return {'assets': 'assetid'}[table]
|
||||
|
||||
|
||||
def apply_plan(connection, plan, approved_packet, acknowledge=None,
|
||||
createdby=None):
|
||||
"""Apply an approved WINC plan, reversibly. Returns the batch id.
|
||||
|
||||
connection a live SQLAlchemy connection in an open transaction.
|
||||
plan the per-row plan from dry_run.analyze (rebuilt fresh).
|
||||
approved_packet the packet a human signed off on.
|
||||
acknowledge set of needs-create group names the human confirmed
|
||||
(e.g. {'businessunit'}); required for every group the
|
||||
packet lists under needscreate.
|
||||
createdby who acknowledged, stamped on the batch.
|
||||
|
||||
Raises WriterError before touching anything if the plan drifted, is
|
||||
blocking, or a needs-create group was not acknowledged.
|
||||
"""
|
||||
acknowledge = set(acknowledge or ())
|
||||
|
||||
# (2)-(3) rebuild the packet and prove nothing drifted since approval.
|
||||
current_packet = build_packet('winc-machine-adoption', plan,
|
||||
approved_packet.get('counts', {}),
|
||||
_PACKET_GROUPS)
|
||||
if not packet_matches_plan(approved_packet, plan):
|
||||
raise WriterError(
|
||||
'plan has changed since approval (hash mismatch) - re-run the '
|
||||
'dry-run and re-approve before importing')
|
||||
|
||||
# (4) never write over an unresolved value.
|
||||
if current_packet.get('blocking'):
|
||||
raise WriterError(
|
||||
'plan is blocking: unresolved values remain - add the missing '
|
||||
'aliases and re-run the dry-run')
|
||||
|
||||
# (5) every needs-create group must be explicitly acknowledged.
|
||||
missing_ack = set(current_packet.get('needscreate', {})) - acknowledge
|
||||
if missing_ack:
|
||||
raise WriterError(
|
||||
'needs-create not acknowledged for: %s - confirm these will be '
|
||||
'created before importing' % ', '.join(sorted(missing_ack)))
|
||||
|
||||
batchid = approved_packet['batchid']
|
||||
journal.open_batch(connection, approved_packet, createdby=createdby)
|
||||
sequence = 1
|
||||
|
||||
# (6a) create acknowledged needs-create business units once each, first, so
|
||||
# the asset updates below can point at them.
|
||||
created_bu = {}
|
||||
for row in plan:
|
||||
if row['businessunit_status'] != NEEDS_CREATE:
|
||||
continue
|
||||
name = row['businessunit_name']
|
||||
key = norm(name)
|
||||
if key not in created_bu:
|
||||
created_bu[key], sequence = _create_businessunit(
|
||||
connection, batchid, sequence, name)
|
||||
|
||||
# (6b) update each in-scope, unambiguous asset.
|
||||
asset_ids = _load_asset_ids(connection)
|
||||
from sqlalchemy import text
|
||||
for row in plan:
|
||||
if row['ambiguous']:
|
||||
continue
|
||||
assetnumber = row['matched_asset']
|
||||
assetid = asset_ids.get(assetnumber)
|
||||
if assetid is None:
|
||||
continue
|
||||
|
||||
current = connection.execute(text(
|
||||
'SELECT name, locationid, businessunitid FROM assets '
|
||||
'WHERE assetid = :assetid'), {'assetid': assetid}).fetchone()
|
||||
cur_name, cur_location, cur_bu = current
|
||||
|
||||
if row['name']:
|
||||
new_name = row['name'][:_ASSET_NAME_MAXLEN]
|
||||
sequence = _update_cell(connection, batchid, sequence, 'assets',
|
||||
assetid, 'name', cur_name, new_name)
|
||||
|
||||
if row['location_status'] == RESOLVED and row['locationid'] != '':
|
||||
sequence = _update_cell(connection, batchid, sequence, 'assets',
|
||||
assetid, 'locationid', cur_location,
|
||||
row['locationid'])
|
||||
|
||||
new_bu = None
|
||||
if row['businessunit_status'] == RESOLVED and row['businessunitid'] != '':
|
||||
new_bu = row['businessunitid']
|
||||
elif row['businessunit_status'] == NEEDS_CREATE:
|
||||
new_bu = created_bu[norm(row['businessunit_name'])]
|
||||
if new_bu is not None:
|
||||
sequence = _update_cell(connection, batchid, sequence, 'assets',
|
||||
assetid, 'businessunitid', cur_bu, new_bu)
|
||||
|
||||
return batchid
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(
|
||||
description='WINC machine adoption writer. Applies an approved plan '
|
||||
'reversibly. Requires --commit to write.')
|
||||
parser.add_argument('--export', required=True,
|
||||
help='Path to the ASSET export CSV.')
|
||||
parser.add_argument('--acknowledge', action='append', default=[],
|
||||
metavar='GROUP',
|
||||
help='Acknowledge a needs-create group (repeatable), '
|
||||
'e.g. --acknowledge businessunit.')
|
||||
parser.add_argument('--createdby', default=None,
|
||||
help='Who is approving this import (stamped on batch).')
|
||||
parser.add_argument('--commit', action='store_true',
|
||||
help='Actually write. Without it, this is a no-op that '
|
||||
'reports what would happen.')
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not os.path.exists(args.export):
|
||||
parser.error('export file not found: %s' % args.export)
|
||||
|
||||
_app, connection = build_app_connection()
|
||||
plan, summary, _lookups = analyze(connection, args.export)
|
||||
packet = build_packet('winc-machine-adoption', plan, summary,
|
||||
_PACKET_GROUPS)
|
||||
|
||||
if not args.commit:
|
||||
print('DRY: rebuilt plan of %d rows, hash %s' % (
|
||||
packet['rowcount'], packet['planhash']))
|
||||
print('DRY: pass --commit with --acknowledge for each needs-create '
|
||||
'group to write. Needs-create groups: %s'
|
||||
% (', '.join(packet.get('needscreate', {})) or 'none'))
|
||||
return 0
|
||||
|
||||
from shopdb.api import db
|
||||
try:
|
||||
batchid = apply_plan(connection, plan, packet,
|
||||
acknowledge=args.acknowledge,
|
||||
createdby=args.createdby)
|
||||
db.session.commit()
|
||||
except WriterError as error:
|
||||
db.session.rollback()
|
||||
print('REFUSED: %s' % error)
|
||||
return 2
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
print('COMMITTED batch %s' % batchid)
|
||||
print(' to undo: python -m scripts.site_imports.winc.undo --batch %s'
|
||||
% batchid)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user