Merge remote-tracking branch 'origin/main' into wilm

# Conflicts:
#	frontend/src/views/settings/SiteSettings.vue

shopdb/core/services/auditretention.py

@@ -0,0 +1,133 @@
+"""Audit log retention.
+
+The audit log is append-only and nothing ever removed a row, so it grows for as
+long as a site runs. That is fine for a year and not fine forever: this site
+reached the point where the log is most of the database and the audit page is
+tens of thousands of pages deep - which also makes it useless, because nobody
+pages through that to find anything.
+
+Retention is OPT-IN, and the default keeps everything. An upgrade that silently
+started deleting a site's audit history would be indefensible; a site that wants
+a limit sets one, and is told exactly how many rows that will remove before it
+happens.
+
+DELETING IN CHUNKS IS NOT AN OPTIMISATION. A single
+`DELETE FROM auditlogs WHERE timestamp < ...` over a million rows builds one
+enormous transaction, holds locks across the table for the duration, and on a
+modest MySQL box either times out or blocks every writer while it runs - which
+on this server means the collectors that are posting inventory. Deleting in
+bounded batches, committing each, keeps every individual statement short.
+
+Chunking is done by SELECTing a page of ids and deleting those. `DELETE ... LIMIT`
+would be simpler but is MySQL-only, and the test suite runs on SQLite.
+"""
+
+from datetime import datetime, timedelta, timezone
+
+from shopdb.extensions import db
+from shopdb.core.models import AuditLog
+
+
+# Big enough that a large purge does not take thousands of round trips, small
+# enough that one statement stays short and a kill mid-purge loses little work.
+CHUNKSIZE = 5000
+
+# A purge triggered from the browser must not run until a proxy gives up on it.
+# The CLI has no such limit and passes None.
+MAXBATCHES_REQUEST = 40
+
+
+def _utcnow():
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+def cutofffor(days):
+    """The timestamp before which rows are older than the retention window."""
+    return _utcnow() - timedelta(days=int(days))
+
+
+def stats(days=None):
+    """Row count, date range, and how many rows a purge at `days` would remove.
+
+    Cheap enough to render on a settings page: three aggregates over an indexed
+    column.
+    """
+    total = db.session.query(db.func.count(AuditLog.auditlogid)).scalar() or 0
+    oldest, newest = db.session.query(
+        db.func.min(AuditLog.timestamp), db.func.max(AuditLog.timestamp)).first()
+
+    expired = None
+    if days:
+        expired = (db.session.query(db.func.count(AuditLog.auditlogid))
+                   .filter(AuditLog.timestamp < cutofffor(days)).scalar() or 0)
+
+    return {
+        'total': total,
+        'oldest': oldest.isoformat() if oldest else None,
+        'newest': newest.isoformat() if newest else None,
+        'retentiondays': int(days) if days else 0,
+        'expired': expired,
+    }
+
+
+def purge(days, dryrun=False, maxbatches=None, chunksize=CHUNKSIZE):
+    """Delete audit rows older than `days`, in committed batches.
+
+    Returns a dict describing what happened, including `remaining`: a purge that
+    stopped early because it hit `maxbatches` says so rather than reporting
+    success over a job it did not finish. A caller that reported "purged" while
+    900k rows were still there would be exactly the kind of quiet half-failure
+    this codebase keeps getting bitten by.
+    """
+    days = int(days or 0)
+    if days <= 0:
+        raise ValueError('retention must be a positive number of days; '
+                         '0 means keep everything, which is not a purge')
+
+    cutoff = cutofffor(days)
+    matched = (db.session.query(db.func.count(AuditLog.auditlogid))
+               .filter(AuditLog.timestamp < cutoff).scalar() or 0)
+
+    if dryrun or not matched:
+        return {'cutoff': cutoff.isoformat(), 'matched': matched, 'deleted': 0,
+                'remaining': matched, 'batches': 0, 'dryrun': bool(dryrun),
+                'complete': not matched}
+
+    deleted = 0
+    batches = 0
+    while True:
+        if maxbatches is not None and batches >= maxbatches:
+            break
+        ids = [row[0] for row in
+               db.session.query(AuditLog.auditlogid)
+               .filter(AuditLog.timestamp < cutoff)
+               .order_by(AuditLog.auditlogid)
+               .limit(chunksize).all()]
+        if not ids:
+            break
+        (db.session.query(AuditLog)
+         .filter(AuditLog.auditlogid.in_(ids))
+         .delete(synchronize_session=False))
+        db.session.commit()
+        deleted += len(ids)
+        batches += 1
+
+    remaining = (db.session.query(db.func.count(AuditLog.auditlogid))
+                 .filter(AuditLog.timestamp < cutoff).scalar() or 0)
+
+    return {'cutoff': cutoff.isoformat(), 'matched': matched, 'deleted': deleted,
+            'remaining': remaining, 'batches': batches, 'dryrun': False,
+            'complete': remaining == 0}
+
+
+def configureddays():
+    """The site's retention setting, or 0 when nothing is set (keep forever)."""
+    from shopdb.core.models import Setting
+    try:
+        raw = Setting.get('audit_retentiondays')
+    except Exception:
+        return 0
+    try:
+        return max(0, int(raw or 0))
+    except (TypeError, ValueError):
+        return 0