Files
shopdb-flask/shopdb/core/api/auditlogs.py
cproudlock 9d96006eae Audit log retention: a way to purge, off by default
The audit log is append-only and nothing ever removed a row, so it grows for as
long as a site runs. This site reached tens of thousands of pages, which is past
the point of being useful: nobody pages through that to find anything.

Retention is OPT-IN and the default keeps everything. An upgrade that silently
started deleting a site's audit history would be indefensible, so
audit_retentiondays defaults to 0 and nothing is removed until somebody sets a
number. The Site & Facility card shows the log's size and exactly how many
entries a purge would delete BEFORE anything is pressed.

Deleting happens in committed batches. That is not an optimisation: one DELETE
over a million rows builds a single enormous transaction and holds locks across
the table while it runs, which on this server means blocking the collectors
posting inventory. The API caps its batch count so the request cannot outlive a
proxy timeout, and a run that stops early reports how much is left rather than
claiming it finished - a purge reporting success over 900k remaining rows would
be the same silent half-failure this codebase keeps getting bitten by.

`flask audit stats` and `flask audit purge` for the server, the latter reporting
by default and deleting only with --apply, with no batch cap because it is not
behind a proxy and a purge that stops halfway leaves the operator guessing.

audit.purge is a separate permission from audit.view. Reading the log and
destroying it are different acts, and plenty of people need only the first.

Chunking selects a page of ids and deletes those rather than using
DELETE ... LIMIT, which is MySQL-only while the suite runs on SQLite.
2026-08-26 15:05:43 -04:00

269 lines
8.9 KiB
Python

"""Audit log API routes."""
from flask import Blueprint, request
from flask_jwt_extended import jwt_required
from shopdb.core.models import AuditLog
from shopdb.extensions import db
from shopdb.utils.responses import success_response, error_response, ErrorCodes
from shopdb.utils.authz import require_permission
auditlogs_bp = Blueprint('auditlogs', __name__)
def _resolve_full_names(ssos):
"""Best-effort SSO -> full name map for audit rows. Mode-aware, guarded,
degrades to {} so the list still renders if the directory is unreachable."""
wanted = {s for s in ssos if s and str(s).isdigit()}
if not wanted:
return {}
names = {}
# Self-hosted directory first (no external dependency).
try:
from plugins.employees.models import DirectoryEmployee
rows = DirectoryEmployee.query.filter(
DirectoryEmployee.sso.in_(wanted)).all()
for emp in rows:
full = f'{(emp.firstname or "").strip()} {(emp.lastname or "").strip()}'.strip()
if full:
names[str(emp.sso)] = full
except Exception:
pass
missing = wanted - set(names)
if not missing:
return names
# External HR directory for anything still unresolved.
try:
from shopdb.utils.employee_db import employee_connection
conn = employee_connection()
placeholders = ','.join(['%s'] * len(missing))
with conn.cursor() as cur:
cur.execute(
'SELECT SSO, First_Name, Last_Name FROM employees '
f'WHERE SSO IN ({placeholders})', tuple(missing))
for emp in cur.fetchall():
full = f'{(emp["First_Name"] or "").strip()} {(emp["Last_Name"] or "").strip()}'.strip()
if full:
names[str(emp['SSO'])] = full
conn.close()
except Exception:
pass
return names
@auditlogs_bp.route('', methods=['GET'])
@jwt_required()
@require_permission('audit.view')
def list_auditlogs():
"""
List audit logs with filtering and pagination.
Query params:
page: Page number (default 1)
perpage: Items per page (default 50, max 200)
action: Filter by action (created, updated, deleted)
entitytype: Filter by entity type
userid: Filter by user ID
search: Search in entityname or username
from_date: Filter from date (ISO format)
to_date: Filter to date (ISO format)
"""
page = request.args.get('page', 1, type=int)
perpage = min(request.args.get('perpage', 50, type=int), 200)
query = AuditLog.query
# Filters
action = request.args.get('action')
if action:
query = query.filter(AuditLog.action == action)
entitytype = request.args.get('entitytype')
if entitytype:
query = query.filter(AuditLog.entitytype == entitytype)
userid = request.args.get('userid', type=int)
if userid:
query = query.filter(AuditLog.userid == userid)
search = request.args.get('search')
if search:
search_term = f'%{search}%'
query = query.filter(
(AuditLog.entityname.ilike(search_term)) |
(AuditLog.username.ilike(search_term))
)
from_date = request.args.get('from_date')
if from_date:
from datetime import datetime
try:
dt = datetime.fromisoformat(from_date.replace('Z', '+00:00'))
query = query.filter(AuditLog.timestamp >= dt)
except ValueError:
pass
to_date = request.args.get('to_date')
if to_date:
from datetime import datetime
try:
dt = datetime.fromisoformat(to_date.replace('Z', '+00:00'))
query = query.filter(AuditLog.timestamp <= dt)
except ValueError:
pass
# Order by most recent first
query = query.order_by(AuditLog.timestamp.desc())
# Paginate
pagination = query.paginate(page=page, per_page=perpage, error_out=False)
rows = [log.to_dict() for log in pagination.items]
# Attach full names (hover tooltip on the SSO); best-effort.
fullnames = _resolve_full_names({r.get('username') for r in rows})
for r in rows:
r['userfullname'] = fullnames.get(str(r.get('username')))
return success_response(
rows,
meta={
'page': page,
'perpage': perpage,
'total': pagination.total,
'pages': pagination.pages
}
)
@auditlogs_bp.route('/entity/<entitytype>/<int:entityid>', methods=['GET'])
@jwt_required()
@require_permission('audit.view')
def get_entity_history(entitytype: str, entityid: int):
"""Get audit history for a specific entity."""
logs = AuditLog.query.filter_by(
entitytype=entitytype,
entityid=entityid
).order_by(AuditLog.timestamp.desc()).all()
return success_response([log.to_dict() for log in logs])
@auditlogs_bp.route('/stats', methods=['GET'])
@jwt_required()
@require_permission('audit.view')
def get_stats():
"""Get audit log statistics."""
from sqlalchemy import func
from datetime import datetime, timedelta, timezone
# Actions by type
actions = db_func_count_by(AuditLog.action)
# Entity types
entities = db_func_count_by(AuditLog.entitytype)
# Recent activity (last 7 days)
week_ago = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=7)
recent_count = AuditLog.query.filter(AuditLog.timestamp >= week_ago).count()
# Most active users (last 7 days)
from shopdb.extensions import db
active_users = db.session.query(
AuditLog.username,
func.count(AuditLog.auditlogid).label('count')
).filter(
AuditLog.timestamp >= week_ago,
AuditLog.username.isnot(None)
).group_by(AuditLog.username).order_by(func.count(AuditLog.auditlogid).desc()).limit(5).all()
return success_response({
'actions': actions,
'entities': entities,
'recentCount': recent_count,
'activeUsers': [{'username': u[0], 'count': u[1]} for u in active_users]
})
def db_func_count_by(column):
"""Helper to count grouped by a column."""
from sqlalchemy import func
from shopdb.extensions import db
results = db.session.query(
column,
func.count().label('count')
).group_by(column).all()
return {r[0]: r[1] for r in results}
@auditlogs_bp.route('/retention', methods=['GET'])
@jwt_required()
@require_permission('audit.view')
def get_retention():
"""Size of the log and what the configured retention would remove.
Drives the Site & Facility card: an admin should see the number of entries a
purge deletes BEFORE pressing anything.
"""
from shopdb.core.services import auditretention
days = auditretention.configureddays()
return success_response(auditretention.stats(days or None))
@auditlogs_bp.route('/purge', methods=['POST'])
@jwt_required()
@require_permission('audit.purge')
def purge_auditlogs():
"""Delete entries older than the retention window.
Separate permission from audit.view: reading the log and destroying it are
not the same act, and plenty of people need the first.
Batch-capped, because this runs inside a request that a proxy will time out.
An incomplete purge returns complete=false and the number still to go rather
than reporting a finished job - the caller re-posts, or runs
`flask audit purge --apply` on the server, which has no cap.
"""
from flask import request as flask_request
from shopdb.core.services import auditretention
body = flask_request.get_json(silent=True) or {}
days = body.get('days')
if days is None:
days = auditretention.configureddays()
try:
days = int(days)
except (TypeError, ValueError):
days = 0
if days <= 0:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'No retention period is configured. Set one in Site & Facility '
'settings before purging.')
dryrun = bool(body.get('dryrun'))
result = auditretention.purge(
days, dryrun=dryrun,
maxbatches=None if dryrun else auditretention.MAXBATCHES_REQUEST)
if dryrun:
message = '{:,} entry(s) would be deleted'.format(result['matched'])
elif result['complete']:
message = 'Deleted {:,} entry(s)'.format(result['deleted'])
else:
message = ('Deleted {:,} entry(s); {:,} still older than the cutoff. '
'Run it again to continue.').format(
result['deleted'], result['remaining'])
# The purge is itself an audited act, and this row postdates the cutoff so
# it survives the purge that created it.
if not dryrun and result['deleted']:
from shopdb.api import audit_log
audit_log(action='deleted', entitytype='AuditLog',
entityname='purge older than {} days'.format(days),
changes={'deleted': result['deleted'],
'cutoff': result['cutoff']})
return success_response(result, message=message)