Make a partly-undone import retryable, and refuse a plan it cannot deliver

shopdb/core/models/__init__.py

@@ -24,7 +24,7 @@ from .auditlog import AuditLog
 from .customfield import CustomField, CustomFieldValue
 from .apitoken import ApiToken
 from .importbatch import (ImportBatch, ImportJournalEntry, BATCH_APPLIED,
-                          BATCH_UNDONE, OP_UPDATE, OP_INSERT)
+                          BATCH_PARTIAL, BATCH_UNDONE, OP_UPDATE, OP_INSERT)
 
 __all__ = [
     # Base
@@ -88,6 +88,7 @@ __all__ = [
     'ImportBatch',
     'ImportJournalEntry',
     'BATCH_APPLIED',
+    'BATCH_PARTIAL',
     'BATCH_UNDONE',
     'OP_UPDATE',
     'OP_INSERT',

shopdb/core/models/base.py

@@ -29,10 +29,37 @@ class BaseModel(db.Model):
     )
     isactive = db.Column(db.Boolean, default=True, nullable=False)
 
+    # Columns the default to_dict NEVER emits, whatever model carries them.
+    #
+    # The default serialiser walks __table__.columns and publishes every one, so
+    # a model that does not override it publishes whatever it has - including
+    # columns added to it later, by someone who was not thinking about this
+    # method. User is the live example: it does not override to_dict, so
+    # User.to_dict() returns passwordhash. Nothing calls it in a response path
+    # today, which is luck rather than design.
+    #
+    # A blanket allowlist would be the stronger rule, but it would also silently
+    # empty the output of all 34 models that rely on this default. This is the
+    # part that can be fixed without breaking anything: name the columns that
+    # must never be published, and let a model add its own.
+    #
+    # A model with something sensitive should still override to_dict and say
+    # exactly what it emits. This is a floor, not permission to skip that.
+    HIDDEN_COLUMNS = frozenset({
+        'passwordhash',
+        'tokenhash',
+        'apikey',
+        'clientsecret',
+        'collectorkey',
+    })
+
     def to_dict(self):
-        """Convert model to dictionary."""
+        """Convert model to dictionary, minus anything secret-bearing."""
+        hidden = self.HIDDEN_COLUMNS
         result = {}
         for c in self.__table__.columns:
+            if c.name in hidden:
+                continue
             value = getattr(self, c.name)
             if isinstance(value, datetime):
                 value = value.isoformat() + 'Z'

shopdb/core/models/importbatch.py

@@ -26,6 +26,12 @@ from shopdb.extensions import db
 # Batch lifecycle.
 BATCH_APPLIED = 'applied'
 BATCH_UNDONE = 'undone'
+# An undo that reverted some entries and hit conflicts on the rest. It is a
+# distinct state because 'undone' is a door that locks: undo_batch returns
+# immediately for an undone batch, so marking a partly-reverted one 'undone'
+# made the leftovers permanently unrevertable - the human resolves the conflict,
+# clicks undo again, and gets a no-op.
+BATCH_PARTIAL = 'partially-undone'
 
 # Journal operation kinds.
 OP_UPDATE = 'update'
@@ -43,7 +49,9 @@ class ImportBatch(db.Model):
     rowcount = db.Column(db.Integer, nullable=False, default=0)
     createdat = db.Column(db.DateTime, nullable=False)
     createdby = db.Column(db.String(100), nullable=True)
-    status = db.Column(db.String(16), nullable=False, default=BATCH_APPLIED)
+    # 32, not 16: BATCH_PARTIAL is exactly 16 characters, so the original
+    # width fitted it with nothing to spare. See migration 7d38.
+    status = db.Column(db.String(32), nullable=False, default=BATCH_APPLIED)
     undoneat = db.Column(db.DateTime, nullable=True)
 
     entries = db.relationship(