shopdb-flask/shopdb/core/services/auditretention.py

"""Audit log retention.

The audit log is append-only and nothing ever removed a row, so it grows for as
long as a site runs. That is fine for a year and not fine forever: this site
reached the point where the log is most of the database and the audit page is
tens of thousands of pages deep - which also makes it useless, because nobody
pages through that to find anything.

Retention is OPT-IN, and the default keeps everything. An upgrade that silently
started deleting a site's audit history would be indefensible; a site that wants
a limit sets one, and is told exactly how many rows that will remove before it
happens.

DELETING IN CHUNKS IS NOT AN OPTIMISATION. A single
`DELETE FROM auditlogs WHERE timestamp < ...` over a million rows builds one
enormous transaction, holds locks across the table for the duration, and on a
modest MySQL box either times out or blocks every writer while it runs - which
on this server means the collectors that are posting inventory. Deleting in
bounded batches, committing each, keeps every individual statement short.

Chunking is done by SELECTing a page of ids and deleting those. `DELETE ... LIMIT`
would be simpler but is MySQL-only, and the test suite runs on SQLite.
"""

from datetime import datetime, timedelta, timezone

from shopdb.extensions import db
from shopdb.core.models import AuditLog


# Big enough that a large purge does not take thousands of round trips, small
# enough that one statement stays short and a kill mid-purge loses little work.
CHUNKSIZE = 5000

# A purge triggered from the browser must not run until a proxy gives up on it.
# The CLI has no such limit and passes None.
MAXBATCHES_REQUEST = 40


def _utcnow():
    return datetime.now(timezone.utc).replace(tzinfo=None)


def cutofffor(days):
    """The timestamp before which rows are older than the retention window."""
    return _utcnow() - timedelta(days=int(days))


def stats(days=None):
    """Row count, date range, and how many rows a purge at `days` would remove.

    Cheap enough to render on a settings page: three aggregates over an indexed
    column.
    """
    total = db.session.query(db.func.count(AuditLog.auditlogid)).scalar() or 0
    oldest, newest = db.session.query(
        db.func.min(AuditLog.timestamp), db.func.max(AuditLog.timestamp)).first()

    expired = None
    if days:
        expired = (db.session.query(db.func.count(AuditLog.auditlogid))
                   .filter(AuditLog.timestamp < cutofffor(days)).scalar() or 0)

    return {
        'total': total,
        'oldest': oldest.isoformat() if oldest else None,
        'newest': newest.isoformat() if newest else None,
        'retentiondays': int(days) if days else 0,
        'expired': expired,
    }


def purge(days, dryrun=False, maxbatches=None, chunksize=CHUNKSIZE):
    """Delete audit rows older than `days`, in committed batches.

    Returns a dict describing what happened, including `remaining`: a purge that
    stopped early because it hit `maxbatches` says so rather than reporting
    success over a job it did not finish. A caller that reported "purged" while
    900k rows were still there would be exactly the kind of quiet half-failure
    this codebase keeps getting bitten by.
    """
    days = int(days or 0)
    if days <= 0:
        raise ValueError('retention must be a positive number of days; '
                         '0 means keep everything, which is not a purge')

    cutoff = cutofffor(days)
    matched = (db.session.query(db.func.count(AuditLog.auditlogid))
               .filter(AuditLog.timestamp < cutoff).scalar() or 0)

    if dryrun or not matched:
        return {'cutoff': cutoff.isoformat(), 'matched': matched, 'deleted': 0,
                'remaining': matched, 'batches': 0, 'dryrun': bool(dryrun),
                'complete': not matched}

    deleted = 0
    batches = 0
    while True:
        if maxbatches is not None and batches >= maxbatches:
            break
        ids = [row[0] for row in
               db.session.query(AuditLog.auditlogid)
               .filter(AuditLog.timestamp < cutoff)
               .order_by(AuditLog.auditlogid)
               .limit(chunksize).all()]
        if not ids:
            break
        (db.session.query(AuditLog)
         .filter(AuditLog.auditlogid.in_(ids))
         .delete(synchronize_session=False))
        db.session.commit()
        deleted += len(ids)
        batches += 1

    remaining = (db.session.query(db.func.count(AuditLog.auditlogid))
                 .filter(AuditLog.timestamp < cutoff).scalar() or 0)

    return {'cutoff': cutoff.isoformat(), 'matched': matched, 'deleted': deleted,
            'remaining': remaining, 'batches': batches, 'dryrun': False,
            'complete': remaining == 0}


def configureddays():
    """The site's retention setting, or 0 when nothing is set (keep forever)."""
    from shopdb.core.models import Setting
    try:
        raw = Setting.get('audit_retentiondays')
    except Exception:
        return 0
    try:
        return max(0, int(raw or 0))
    except (TypeError, ValueError):
        return 0