Import the wilm agent store, WINC adoption work and batches CLI, with four fixes

shopdb/core/api/settings.py

@@ -595,6 +595,134 @@ def import_asset_csvs():
     })
 
 
+def _winc_upload_text():
+    """Read the single uploaded WINC export as text, or (None, error_response)."""
+    uploads = request.files.getlist('files')
+    if not uploads:
+        return None, error_response(ErrorCodes.VALIDATION_ERROR,
+                                    'No export file was uploaded')
+    upload = uploads[0]
+    name = upload.filename or ''
+    if not name.lower().endswith('.csv'):
+        return None, error_response(ErrorCodes.VALIDATION_ERROR,
+                                    'The WINC export must be a .csv file')
+    try:
+        return upload.read().decode('utf-8-sig'), None
+    except (UnicodeDecodeError, OSError):
+        return None, error_response(ErrorCodes.VALIDATION_ERROR,
+                                    'Could not read %s as UTF-8 text' % name)
+
+
+@settings_bp.route('/asset-import/winc/preview', methods=['POST'])
+@require_role('admin')
+def winc_import_preview():
+    """Read-only WINC analysis: report what a commit would do. Writes nothing.
+
+    WINC (a Maximo-style asset dump) is not a canonical-CSV source: it updates
+    existing machines only, needs acknowledgment before creating business units,
+    and is reversible. This returns the approval packet the UI renders - counts,
+    the business units it would create, and any unresolved values that block it.
+    """
+    from shopdb.core.services.wincimport import preview, WincImportError
+
+    export_text, failure = _winc_upload_text()
+    if failure is not None:
+        return failure
+    try:
+        report = preview(export_text)
+    except WincImportError as exc:
+        return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
+    finally:
+        # A preview must leave nothing behind even though it only read: the
+        # analyzer runs against the request-scoped session.
+        db.session.rollback()
+    return success_response(report)
+
+
+@settings_bp.route('/asset-import/winc/commit', methods=['POST'])
+@require_role('admin')
+def winc_import_commit():
+    """Apply a WINC plan reversibly. Requires acknowledgment for needs-create.
+
+    The 'acknowledge' form field is a comma-separated list of needs-create group
+    names the admin confirmed (e.g. 'businessunit'). The whole write is one
+    transaction recorded to the import journal, so it can be undone later.
+    """
+    from shopdb.core.services.wincimport import commit as winc_commit
+    from shopdb.core.services.wincimport import WincImportError
+
+    export_text, failure = _winc_upload_text()
+    if failure is not None:
+        return failure
+
+    acknowledge = [
+        part.strip() for part in
+        (request.form.get('acknowledge') or '').split(',') if part.strip()]
+
+    # The hash the PREVIEW returned. Required here, because without it the
+    # writer's drift check compares a freshly built packet against the plan it
+    # was built from and can never fail - so "the admin approved this plan"
+    # meant only "the admin ticked these group names".
+    planhash = (request.form.get('planhash') or '').strip()
+    if not planhash:
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'planhash is required: send the value the preview returned, so the '
+            'commit can prove it is applying the plan that was approved.')
+
+    try:
+        report = winc_commit(export_text, acknowledge=acknowledge,
+                             createdby=get_jwt_identity(), planhash=planhash)
+        AuditLog.log(
+            'imported', 'SiteAssets', entityname='winc-machine-adoption',
+            changes={'batchid': report.get('batchid'),
+                     'rows': report.get('rowcount')})
+        db.session.commit()
+    except WincImportError as exc:
+        db.session.rollback()
+        return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
+    except Exception:
+        db.session.rollback()
+        raise
+    return success_response(report)
+
+
+@settings_bp.route('/asset-import/winc/batches', methods=['GET'])
+@require_role('admin')
+def winc_import_batches():
+    """List applied/undone WINC import batches, newest first (for the undo UI)."""
+    from shopdb.core.services.wincimport import list_batches
+
+    return success_response({'batches': list_batches()})
+
+
+@settings_bp.route('/asset-import/winc/undo', methods=['POST'])
+@require_role('admin')
+def winc_import_undo():
+    """Reverse an applied WINC import batch by id. Conservative and reversible."""
+    from shopdb.core.services.wincimport import undo as winc_undo
+
+    payload = request.get_json(silent=True) or {}
+    batchid = (payload.get('batchid') or request.form.get('batchid') or '').strip()
+    if not batchid:
+        return error_response(ErrorCodes.VALIDATION_ERROR, 'A batchid is required')
+
+    try:
+        result = winc_undo(batchid)
+        AuditLog.log(
+            'undone', 'SiteAssets', entityname='winc-machine-adoption',
+            changes={'batchid': batchid, 'reverted': result['reverted'],
+                     'conflicts': len(result['conflicts'])})
+        db.session.commit()
+    except ValueError as exc:
+        db.session.rollback()
+        return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
+    except Exception:
+        db.session.rollback()
+        raise
+    return success_response(result)
+
+
 @settings_bp.route('', methods=['GET'])
 @jwt_required(optional=True)
 def list_settings():

shopdb/core/models/__init__.py

@@ -23,6 +23,8 @@ from .setting import Setting
 from .auditlog import AuditLog
 from .customfield import CustomField, CustomFieldValue
 from .apitoken import ApiToken
+from .importbatch import (ImportBatch, ImportJournalEntry, BATCH_APPLIED,
+                          BATCH_UNDONE, OP_UPDATE, OP_INSERT)
 
 __all__ = [
     # Base
@@ -82,4 +84,11 @@ __all__ = [
     'CustomFieldValue',
     # Personal API tokens
     'ApiToken',
+    # Import batch + journal (reversible bulk imports)
+    'ImportBatch',
+    'ImportJournalEntry',
+    'BATCH_APPLIED',
+    'BATCH_UNDONE',
+    'OP_UPDATE',
+    'OP_INSERT',
 ]

shopdb/core/models/importbatch.py

@@ -0,0 +1,86 @@
+"""Import batch + journal models (reversible bulk imports).
+
+PLATFORM-LEVEL, IMPORT-AGNOSTIC. These two tables let any bulk-import writer
+record exactly what it changed so the change can be reversed later. Nothing here
+knows about Maximo, WINC, machines, or any one source: an importer records
+primitive mutations (set column X on row Y, or insert row Y) tied to a batch id,
+and a generic undo replays them in reverse.
+
+importbatches       one row per approved write - the identity that approval, the
+                    write, and the eventual undo all reference (the same value
+                    the approval packet calls batchid).
+importjournalentries one row per primitive mutation the writer made, ordered so
+                    undo can reverse them (inserts removed after the updates that
+                    pointed at them are reverted).
+
+The undo contract, kept deliberately conservative:
+  - update is reverted only if the cell still holds the value we wrote; a value a
+    human changed since is left as-is and reported (never clobbered).
+  - insert is hard-deleted, but ONLY the row this batch created and only when
+    nothing outside the batch still references it.
+"""
+
+from shopdb.extensions import db
+
+
+# Batch lifecycle.
+BATCH_APPLIED = 'applied'
+BATCH_UNDONE = 'undone'
+
+# Journal operation kinds.
+OP_UPDATE = 'update'
+OP_INSERT = 'insert'
+
+
+class ImportBatch(db.Model):
+    """One approved import write. The batchid is the approval packet's batchid,
+    carried through write and undo so all three reference one identity."""
+    __tablename__ = 'importbatches'
+
+    batchid = db.Column(db.String(36), primary_key=True)
+    kind = db.Column(db.String(64), nullable=False)
+    planhash = db.Column(db.String(64), nullable=False)
+    rowcount = db.Column(db.Integer, nullable=False, default=0)
+    createdat = db.Column(db.DateTime, nullable=False)
+    createdby = db.Column(db.String(100), nullable=True)
+    status = db.Column(db.String(16), nullable=False, default=BATCH_APPLIED)
+    undoneat = db.Column(db.DateTime, nullable=True)
+
+    entries = db.relationship(
+        'ImportJournalEntry',
+        backref='batch',
+        cascade='all, delete-orphan',
+        order_by='ImportJournalEntry.sequence',
+    )
+
+    def __repr__(self):
+        return f"<ImportBatch {self.kind} {self.batchid} {self.status}>"
+
+
+class ImportJournalEntry(db.Model):
+    """One reversible primitive mutation made by an import writer."""
+    __tablename__ = 'importjournalentries'
+
+    # SQLite only autoincrements a plain INTEGER PRIMARY KEY, not BIGINT, so the
+    # test database needs the Integer variant while MySQL keeps BIGINT for range.
+    entryid = db.Column(
+        db.BigInteger().with_variant(db.Integer, 'sqlite'),
+        primary_key=True)
+    batchid = db.Column(
+        db.String(36),
+        db.ForeignKey('importbatches.batchid', ondelete='CASCADE'),
+        nullable=False,
+        index=True,
+    )
+    sequence = db.Column(db.Integer, nullable=False)
+    operation = db.Column(db.String(16), nullable=False)
+    targettable = db.Column(db.String(64), nullable=False)
+    targetid = db.Column(db.BigInteger, nullable=False)
+    columnname = db.Column(db.String(64), nullable=True)
+    oldvalue = db.Column(db.Text, nullable=True)
+    newvalue = db.Column(db.Text, nullable=True)
+    isreverted = db.Column(db.Boolean, nullable=False, default=False)
+
+    def __repr__(self):
+        return (f"<ImportJournalEntry {self.operation} "
+                f"{self.targettable}.{self.targetid}>")

shopdb/core/services/wincimport.py

@@ -0,0 +1,154 @@
+"""WINC machine-adoption import service (preview / commit / undo / list).
+
+Web-facing wrapper over the validated WINC CLI helpers. The CLI in
+scripts/site_imports/winc drives the same analyzer, writer, and journal; this
+module adapts them to a Flask request: it works on the request-scoped
+db.session connection (so the route owns the transaction and commit) and returns
+plain JSON-serialisable dicts for the API.
+
+WINC is NOT a source adapter like Azure/Thor. It never creates assets - it
+updates existing machines only, requires acknowledgment before creating business
+units, and records every change to the reversible import journal. That contract
+lives here and in the writer, not in the generic canonical-CSV importer.
+"""
+
+from shopdb.extensions import db
+from scripts.site_imports import journal
+from scripts.site_imports.approval import build_packet
+from scripts.site_imports.winc import writer as winc_writer
+from scripts.site_imports.winc.dry_run import analyze_rows, parse_export_text
+
+
+class WincImportError(Exception):
+    """An uploaded export could not be analyzed or applied as WINC."""
+
+
+def _connection():
+    return db.session.connection()
+
+
+def _plan_and_packet(export_text):
+    """Parse an uploaded export, build the plan and the approval packet."""
+    rows = parse_export_text(export_text)
+    if not rows:
+        raise WincImportError('the uploaded file had no rows')
+    if 'assetnum' not in rows[0]:
+        raise WincImportError(
+            'this does not look like a WINC asset export (no assetnum column)')
+
+    plan, summary, _lookups = analyze_rows(_connection(), rows)
+    packet = build_packet('winc-machine-adoption', plan, summary,
+                          winc_writer._PACKET_GROUPS)
+    return plan, summary, packet
+
+
+def _report(summary, packet):
+    """Shape the packet + summary into the JSON the UI renders."""
+    return {
+        'kind': packet['kind'],
+        'planhash': packet['planhash'],
+        'rowcount': packet['rowcount'],
+        'summary': dict(summary),
+        'needscreate': {group: dict(items)
+                        for group, items in packet.get('needscreate', {}).items()},
+        'unresolved': {group: dict(items)
+                       for group, items in packet.get('unresolved', {}).items()},
+        'blocking': packet.get('blocking', False),
+    }
+
+
+def preview(export_text):
+    """Read-only: analyze the export and return the report the UI shows.
+
+    Writes nothing. The whole point is to surface what a commit WOULD do -
+    counts, the business units it would create (needs acknowledgment), and any
+    unresolved values that block the import.
+    """
+    _plan, summary, packet = _plan_and_packet(export_text)
+    report = _report(summary, packet)
+    report['committed'] = False
+    return report
+
+
+def commit(export_text, acknowledge=None, createdby=None, planhash=None):
+    """Apply the plan reversibly inside the request transaction.
+
+    Rebuilds the plan from the same upload, re-checks it against a freshly built
+    packet (drift, blocking, acknowledgment) and writes through the journal. The
+    caller owns the commit/rollback of db.session. Returns the report plus the
+    batchid on success.
+
+    planhash is the one the PREVIEW returned. It is what makes the approval mean
+    anything: the packet handed to the writer is built from this same upload, so
+    the writer's drift check compared a hash against itself and could never fail.
+    An admin previewing file A and committing file B - or the same file after the
+    locations or aliases moved underneath it - got whatever the second plan said,
+    gated only by the needs-create group NAMES they happened to tick.
+
+    Passing it is optional so the CLI path, which builds and applies in one go,
+    is unaffected. The web route requires it.
+
+    Raises WincImportError (mapped to a 400 by the route) when the plan is
+    blocking, a needs-create group was not acknowledged, or the plan no longer
+    matches the one that was approved.
+    """
+    plan, summary, packet = _plan_and_packet(export_text)
+    if planhash is not None and packet['planhash'] != planhash:
+        raise WincImportError(
+            'This plan is not the one that was previewed - the upload or the '
+            'data behind it changed. Preview again and re-check what it would '
+            'create before committing.')
+    try:
+        batchid = winc_writer.apply_plan(
+            _connection(), plan, packet,
+            acknowledge=acknowledge, createdby=createdby)
+    except winc_writer.WriterError as exc:
+        raise WincImportError(str(exc))
+
+    report = _report(summary, packet)
+    report['committed'] = True
+    report['batchid'] = batchid
+    return report
+
+
+def undo(batchid):
+    """Reverse an applied batch. Caller owns the transaction. Returns a dict."""
+    result = journal.undo_batch(_connection(), batchid,
+                                references=winc_writer._UNDO_REFERENCES)
+    return {
+        'batchid': result.batchid,
+        'reverted': result.reverted,
+        'conflicts': [
+            {'operation': conflict.operation, 'table': conflict.table,
+             'targetid': conflict.targetid, 'column': conflict.column,
+             'reason': conflict.reason}
+            for conflict in result.conflicts
+        ],
+    }
+
+
+def list_batches(limit=50):
+    """Applied/undone WINC batches, newest first, for the UI's undo list."""
+    from sqlalchemy import text
+
+    def _iso(value):
+        # PyMySQL hands back datetime objects; SQLite (tests) hands back the
+        # stored ISO string. Normalise both to an ISO string with a Z suffix.
+        if value is None:
+            return None
+        if hasattr(value, 'isoformat'):
+            return value.isoformat() + 'Z'
+        return str(value)
+
+    rows = _connection().execute(text(
+        'SELECT batchid, kind, rowcount, createdat, createdby, status, '
+        'undoneat FROM importbatches WHERE kind = :kind '
+        'ORDER BY createdat DESC LIMIT :limit'),
+        {'kind': 'winc-machine-adoption', 'limit': limit}).fetchall()
+    return [
+        {'batchid': r[0], 'kind': r[1], 'rowcount': r[2],
+         'createdat': _iso(r[3]),
+         'createdby': r[4], 'status': r[5],
+         'undoneat': _iso(r[6])}
+        for r in rows
+    ]