134 lines
5.0 KiB
Python
134 lines
5.0 KiB
Python
"""Audit log retention.
|
|
|
|
The audit log is append-only and nothing ever removed a row, so it grows for as
|
|
long as a site runs. That is fine for a year and not fine forever: this site
|
|
reached the point where the log is most of the database and the audit page is
|
|
tens of thousands of pages deep - which also makes it useless, because nobody
|
|
pages through that to find anything.
|
|
|
|
Retention is OPT-IN, and the default keeps everything. An upgrade that silently
|
|
started deleting a site's audit history would be indefensible; a site that wants
|
|
a limit sets one, and is told exactly how many rows that will remove before it
|
|
happens.
|
|
|
|
DELETING IN CHUNKS IS NOT AN OPTIMISATION. A single
|
|
`DELETE FROM auditlogs WHERE timestamp < ...` over a million rows builds one
|
|
enormous transaction, holds locks across the table for the duration, and on a
|
|
modest MySQL box either times out or blocks every writer while it runs - which
|
|
on this server means the collectors that are posting inventory. Deleting in
|
|
bounded batches, committing each, keeps every individual statement short.
|
|
|
|
Chunking is done by SELECTing a page of ids and deleting those. `DELETE ... LIMIT`
|
|
would be simpler but is MySQL-only, and the test suite runs on SQLite.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from shopdb.extensions import db
|
|
from shopdb.core.models import AuditLog
|
|
|
|
|
|
# Big enough that a large purge does not take thousands of round trips, small
|
|
# enough that one statement stays short and a kill mid-purge loses little work.
|
|
CHUNKSIZE = 5000
|
|
|
|
# A purge triggered from the browser must not run until a proxy gives up on it.
|
|
# The CLI has no such limit and passes None.
|
|
MAXBATCHES_REQUEST = 40
|
|
|
|
|
|
def _utcnow():
|
|
return datetime.now(timezone.utc).replace(tzinfo=None)
|
|
|
|
|
|
def cutofffor(days):
|
|
"""The timestamp before which rows are older than the retention window."""
|
|
return _utcnow() - timedelta(days=int(days))
|
|
|
|
|
|
def stats(days=None):
|
|
"""Row count, date range, and how many rows a purge at `days` would remove.
|
|
|
|
Cheap enough to render on a settings page: three aggregates over an indexed
|
|
column.
|
|
"""
|
|
total = db.session.query(db.func.count(AuditLog.auditlogid)).scalar() or 0
|
|
oldest, newest = db.session.query(
|
|
db.func.min(AuditLog.timestamp), db.func.max(AuditLog.timestamp)).first()
|
|
|
|
expired = None
|
|
if days:
|
|
expired = (db.session.query(db.func.count(AuditLog.auditlogid))
|
|
.filter(AuditLog.timestamp < cutofffor(days)).scalar() or 0)
|
|
|
|
return {
|
|
'total': total,
|
|
'oldest': oldest.isoformat() if oldest else None,
|
|
'newest': newest.isoformat() if newest else None,
|
|
'retentiondays': int(days) if days else 0,
|
|
'expired': expired,
|
|
}
|
|
|
|
|
|
def purge(days, dryrun=False, maxbatches=None, chunksize=CHUNKSIZE):
|
|
"""Delete audit rows older than `days`, in committed batches.
|
|
|
|
Returns a dict describing what happened, including `remaining`: a purge that
|
|
stopped early because it hit `maxbatches` says so rather than reporting
|
|
success over a job it did not finish. A caller that reported "purged" while
|
|
900k rows were still there would be exactly the kind of quiet half-failure
|
|
this codebase keeps getting bitten by.
|
|
"""
|
|
days = int(days or 0)
|
|
if days <= 0:
|
|
raise ValueError('retention must be a positive number of days; '
|
|
'0 means keep everything, which is not a purge')
|
|
|
|
cutoff = cutofffor(days)
|
|
matched = (db.session.query(db.func.count(AuditLog.auditlogid))
|
|
.filter(AuditLog.timestamp < cutoff).scalar() or 0)
|
|
|
|
if dryrun or not matched:
|
|
return {'cutoff': cutoff.isoformat(), 'matched': matched, 'deleted': 0,
|
|
'remaining': matched, 'batches': 0, 'dryrun': bool(dryrun),
|
|
'complete': not matched}
|
|
|
|
deleted = 0
|
|
batches = 0
|
|
while True:
|
|
if maxbatches is not None and batches >= maxbatches:
|
|
break
|
|
ids = [row[0] for row in
|
|
db.session.query(AuditLog.auditlogid)
|
|
.filter(AuditLog.timestamp < cutoff)
|
|
.order_by(AuditLog.auditlogid)
|
|
.limit(chunksize).all()]
|
|
if not ids:
|
|
break
|
|
(db.session.query(AuditLog)
|
|
.filter(AuditLog.auditlogid.in_(ids))
|
|
.delete(synchronize_session=False))
|
|
db.session.commit()
|
|
deleted += len(ids)
|
|
batches += 1
|
|
|
|
remaining = (db.session.query(db.func.count(AuditLog.auditlogid))
|
|
.filter(AuditLog.timestamp < cutoff).scalar() or 0)
|
|
|
|
return {'cutoff': cutoff.isoformat(), 'matched': matched, 'deleted': deleted,
|
|
'remaining': remaining, 'batches': batches, 'dryrun': False,
|
|
'complete': remaining == 0}
|
|
|
|
|
|
def configureddays():
|
|
"""The site's retention setting, or 0 when nothing is set (keep forever)."""
|
|
from shopdb.core.models import Setting
|
|
try:
|
|
raw = Setting.get('audit_retentiondays')
|
|
except Exception:
|
|
return 0
|
|
try:
|
|
return max(0, int(raw or 0))
|
|
except (TypeError, ValueError):
|
|
return 0
|