Make a partly-undone import retryable, and refuse a plan it cannot deliver

shopdb/core/api/settings.py

@@ -701,6 +701,7 @@ def winc_import_batches():
 def winc_import_undo():
     """Reverse an applied WINC import batch by id. Conservative and reversible."""
     from shopdb.core.services.wincimport import undo as winc_undo
+    from shopdb.core.services.wincimport import WincImportError
 
     payload = request.get_json(silent=True) or {}
     batchid = (payload.get('batchid') or request.form.get('batchid') or '').strip()
@@ -714,7 +715,10 @@ def winc_import_undo():
             changes={'batchid': batchid, 'reverted': result['reverted'],
                      'conflicts': len(result['conflicts'])})
         db.session.commit()
-    except ValueError as exc:
+    except (ValueError, WincImportError) as exc:
+        # WincImportError too: refusing a batch of another importer's kind is a
+        # bad request, not a server fault. Preview and commit already catch it;
+        # this route did not, so the refusal surfaced as a 500.
         db.session.rollback()
         return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
     except Exception:

shopdb/core/models/__init__.py

@@ -24,7 +24,7 @@ from .auditlog import AuditLog
 from .customfield import CustomField, CustomFieldValue
 from .apitoken import ApiToken
 from .importbatch import (ImportBatch, ImportJournalEntry, BATCH_APPLIED,
-                          BATCH_UNDONE, OP_UPDATE, OP_INSERT)
+                          BATCH_PARTIAL, BATCH_UNDONE, OP_UPDATE, OP_INSERT)
 
 __all__ = [
     # Base
@@ -88,6 +88,7 @@ __all__ = [
     'ImportBatch',
     'ImportJournalEntry',
     'BATCH_APPLIED',
+    'BATCH_PARTIAL',
     'BATCH_UNDONE',
     'OP_UPDATE',
     'OP_INSERT',

shopdb/core/models/base.py

@@ -29,10 +29,37 @@ class BaseModel(db.Model):
     )
     isactive = db.Column(db.Boolean, default=True, nullable=False)
 
+    # Columns the default to_dict NEVER emits, whatever model carries them.
+    #
+    # The default serialiser walks __table__.columns and publishes every one, so
+    # a model that does not override it publishes whatever it has - including
+    # columns added to it later, by someone who was not thinking about this
+    # method. User is the live example: it does not override to_dict, so
+    # User.to_dict() returns passwordhash. Nothing calls it in a response path
+    # today, which is luck rather than design.
+    #
+    # A blanket allowlist would be the stronger rule, but it would also silently
+    # empty the output of all 34 models that rely on this default. This is the
+    # part that can be fixed without breaking anything: name the columns that
+    # must never be published, and let a model add its own.
+    #
+    # A model with something sensitive should still override to_dict and say
+    # exactly what it emits. This is a floor, not permission to skip that.
+    HIDDEN_COLUMNS = frozenset({
+        'passwordhash',
+        'tokenhash',
+        'apikey',
+        'clientsecret',
+        'collectorkey',
+    })
+
     def to_dict(self):
-        """Convert model to dictionary."""
+        """Convert model to dictionary, minus anything secret-bearing."""
+        hidden = self.HIDDEN_COLUMNS
         result = {}
         for c in self.__table__.columns:
+            if c.name in hidden:
+                continue
             value = getattr(self, c.name)
             if isinstance(value, datetime):
                 value = value.isoformat() + 'Z'

shopdb/core/models/importbatch.py

@@ -26,6 +26,12 @@ from shopdb.extensions import db
 # Batch lifecycle.
 BATCH_APPLIED = 'applied'
 BATCH_UNDONE = 'undone'
+# An undo that reverted some entries and hit conflicts on the rest. It is a
+# distinct state because 'undone' is a door that locks: undo_batch returns
+# immediately for an undone batch, so marking a partly-reverted one 'undone'
+# made the leftovers permanently unrevertable - the human resolves the conflict,
+# clicks undo again, and gets a no-op.
+BATCH_PARTIAL = 'partially-undone'
 
 # Journal operation kinds.
 OP_UPDATE = 'update'
@@ -43,7 +49,9 @@ class ImportBatch(db.Model):
     rowcount = db.Column(db.Integer, nullable=False, default=0)
     createdat = db.Column(db.DateTime, nullable=False)
     createdby = db.Column(db.String(100), nullable=True)
-    status = db.Column(db.String(16), nullable=False, default=BATCH_APPLIED)
+    # 32, not 16: BATCH_PARTIAL is exactly 16 characters, so the original
+    # width fitted it with nothing to spare. See migration 7d38.
+    status = db.Column(db.String(32), nullable=False, default=BATCH_APPLIED)
     undoneat = db.Column(db.DateTime, nullable=True)
 
     entries = db.relationship(

shopdb/core/services/wincimport.py

@@ -12,6 +12,8 @@ units, and records every change to the reversible import journal. That contract
 lives here and in the writer, not in the generic canonical-CSV importer.
 """
 
+import sqlalchemy as sa
+
 from shopdb.extensions import db
 from scripts.site_imports import journal
 from scripts.site_imports.approval import build_packet
@@ -38,7 +40,8 @@ def _plan_and_packet(export_text):
 
     plan, summary, _lookups = analyze_rows(_connection(), rows)
     packet = build_packet('winc-machine-adoption', plan, summary,
-                          winc_writer._PACKET_GROUPS)
+                          winc_writer._PACKET_GROUPS,
+                          creatable=winc_writer._CREATABLE_GROUPS)
     return plan, summary, packet
 
 
@@ -53,6 +56,11 @@ def _report(summary, packet):
                         for group, items in packet.get('needscreate', {}).items()},
         'unresolved': {group: dict(items)
                        for group, items in packet.get('unresolved', {}).items()},
+        # Carried separately from needscreate: both are "rows that do not
+        # exist", but only this one cannot be acknowledged away, and the page
+        # has to say WHY it is blocked rather than only that it is.
+        'uncreatable': {group: dict(items)
+                        for group, items in packet.get('uncreatable', {}).items()},
         'blocking': packet.get('blocking', False),
     }
 
@@ -111,8 +119,27 @@ def commit(export_text, acknowledge=None, createdby=None, planhash=None):
     return report
 
 
+IMPORT_KIND = 'winc-machine-adoption'
+
+
 def undo(batchid):
-    """Reverse an applied batch. Caller owns the transaction. Returns a dict."""
+    """Reverse an applied batch. Caller owns the transaction. Returns a dict.
+
+    Refuses a batch of another kind. The journal is a general primitive and
+    undo_batch does not filter on kind, so without this check the first other
+    importer's batches would be reversible through THIS endpoint - carrying
+    WINC's references map, which lists none of that importer's tables, so the
+    reference guard would pass on everything and the deletes would be
+    unconditional.
+    """
+    kind = _connection().execute(
+        sa.text('SELECT kind FROM importbatches WHERE batchid = :batchid'),
+        {'batchid': batchid}).scalar()
+    if kind is not None and kind != IMPORT_KIND:
+        raise WincImportError(
+            'batch %s was created by a different importer (%s) and cannot be '
+            'reversed here' % (batchid, kind))
+
     result = journal.undo_batch(_connection(), batchid,
                                 references=winc_writer._UNDO_REFERENCES)
     return {