shopdb-flask/plugins/endpoints/api/enrollment.py

"""Per-PC endpoint credentials: enrollment, administration and fleet migration.

Agent-facing:
  POST /enroll                     issue a credential or record a pending request
  POST /collect                    the generic collector ingest, under this prefix

Admin (endpoints.fleet to read, admin role to change):
  GET  /enrollment                 credential state per computer + requests
  POST /enrollment/requests/<id>/approve | reject
  POST /enrollment/assets/<assetid>/revoke          body {"blockreenroll", "reason"}
  POST /enrollment/assets/<assetid>/allow-reenroll  lift a re-enrollment block
  PUT  /enrollment/settings
  GET  /enrollment/migration       readiness of every computer for agent 1.20
  POST /enrollment/migration/wave  push the agent upgrade to a batch of PCs

A PC moves onto its own credential by running an agent that enrolls. The
migration surface computes, per computer, whether the existing agent upgrade
can be pushed to it, and pushes it in waves through the same enqueue path the
single-asset agentupgrade route uses.
"""

import hmac
import re
from collections import Counter
from datetime import datetime, timedelta, timezone

from flask import request
from flask_jwt_extended import current_user

from shopdb.api import (
    db, Asset, AuditLog, Script, error_response, success_response, ErrorCodes,
    require_permission, require_role, collector_authorized, collector_ingest,
    client_ip, get_pagination_params,
)

from ..commands import (
    ACTIVE_LOCK_FOR, ALLOWED_ACTIONS_FOR, agent_meets_minimum,
    minimum_agent_version_for, parse_agent_version,
)
from ..models import (
    EndpointCommand, EndpointCredential, EndpointEnrollRequest,
    EndpointMigrationWave, EndpointState,
)
from . import agentauth
from .routes import endpoints_bp

# First agent that enrolls itself.
CREDENTIAL_AGENT_VERSION = '1.20.0'

CODE_ENROLL_NONCE_MISMATCH = 'ENROLL_NONCE_MISMATCH'

_NONCE_PATTERN = re.compile(r'^[A-Za-z0-9_-]{32,128}$')

REASON_UNKNOWN_HOSTNAME = 'unknown hostname'
REASON_DUPLICATE_HOSTNAME = 'duplicate hostname'
REASON_NO_SERIAL = 'no serial on record'
REASON_SERIAL_MISMATCH = 'serial mismatch'
REASON_ALREADY_ENROLLED = 'already enrolled'
REASON_AWAITING_APPROVAL = 'awaiting approval'
REASON_NONCE_CHANGED = 'nonce changed'
REASON_REJECTED = 'rejected'
REASON_REENROLL_BLOCKED = 'reenroll blocked'

MIGRATION_STATES = ('enrolled', 'pending', 'upgrading', 'ready', 'blocked',
                    'notreporting')
ENROLLMENT_STATES = ('enrolled', 'notenrolled', 'pending', 'revoked')

BLOCKER_OFFLINE = 'offline'
BLOCKER_NO_SERIAL = 'no serial on record'
BLOCKER_DUPLICATE_HOSTNAME = 'duplicate hostname'
BLOCKER_LOCKED = 'commands locked'
BLOCKER_NO_PACKAGE = 'no agent package in store'
BLOCKER_TOO_OLD = 'agent too old for remote upgrade'
BLOCKER_POLICY = 'run_script not permitted by policy'
BLOCKER_AWAITING_ENROLLMENT = 'awaiting enrollment'
BLOCKER_REENROLL_BLOCKED = 'reenroll blocked'
BLOCKER_CERT_UNTRUSTED = 'server certificate not trusted by this PC'

# How agents up to 1.19 word a download refused over the server certificate.
# Only a PC's own recorded failure shows it; the server cannot read a trust store.
_TLS_TRUST_FAILURES = (
    'ssl connection could not be established',
    'untrustedroot',
    'remotecertificatechainerrors',
    'the remote certificate is invalid',
    'could not establish trust relationship',
)

_OFFLINE_AFTER = timedelta(hours=24)
_WAVE_MAXIMUM = 100
_RECENT_REQUESTS = 50


def _naive_utcnow():
    return datetime.now(timezone.utc).replace(tzinfo=None)


def _utciso(value):
    return value.isoformat() + 'Z' if value else None


def _username():
    return getattr(current_user, 'username', None)


def _computer_model():
    from plugins.computers.models import Computer
    return Computer


# ---------------------------------------------------------------------------
# Agent-facing
# ---------------------------------------------------------------------------

@endpoints_bp.route('/collect', methods=['POST'])
def agent_collect():
    """Agent telemetry ingest. Same handler as POST /api/collector/endpoints."""
    return collector_ingest('endpoints')


def _pending(row, reason, matchedassetid=None):
    row.status = EndpointEnrollRequest.STATUS_PENDING
    row.reason = reason
    row.matchedassetid = matchedassetid
    db.session.commit()
    return success_response(
        {'status': 'pending', 'requestid': row.requestid, 'reason': reason},
        http_code=202)


def issue_credential(assetid, enrolledvia):
    """Issue (or re-issue after revoke) the credential for one asset.

    Returns the secret. Only its hash is stored. Caller commits.
    """
    secret = agentauth.generate_credential()
    row = EndpointCredential.query.filter_by(assetid=assetid).first()
    if row is None:
        row = EndpointCredential(assetid=assetid)
        db.session.add(row)
    row.credentialhash = agentauth.hash_credential(secret)
    row.credentialprefix = secret[:agentauth.DISPLAY_PREFIX_LENGTH]
    row.enrolledvia = enrolledvia
    row.issuedat = _naive_utcnow()
    row.lastusedat = None
    row.revokedat = None
    row.revokedby = None
    return secret, row


def _issue(row, assetid, enrolledvia):
    secret, credential = issue_credential(assetid, enrolledvia)
    row.status = EndpointEnrollRequest.STATUS_ISSUED
    row.matchedassetid = assetid
    row.reason = None
    AuditLog.log('created', 'EndpointCredential', entityid=assetid,
                 entityname=row.hostname,
                 details='endpoint credential {} issued ({}), request {}'.format(
                     credential.credentialprefix, enrolledvia, row.requestid))
    db.session.commit()
    return success_response(
        {'status': 'enrolled', 'credential': secret, 'assetid': assetid},
        message='Endpoint enrolled', http_code=201)


def _text_field(payload, name, maximum, required):
    value = payload.get(name)
    if value is None:
        value = ''
    if not isinstance(value, str):
        return None, f'{name} must be a string'
    value = value.strip()
    if required and not value:
        return None, f'{name} is required'
    if len(value) > maximum:
        return None, f'{name} must be at most {maximum} characters'
    return value, None


@endpoints_bp.route('/enroll', methods=['POST'])
def enroll_endpoint():
    """A PC asks for its own credential.

    Shared-key authenticated while plugin.endpoints.sharedkeyallowed is on.
    With it off the route takes no auth and only hands out a credential an
    admin already approved for this exact hostname, serial and nonce.
    """
    sharedkeyauthenticated = False
    if agentauth.shared_key_allowed():
        error = collector_authorized('endpoints')
        if error is not None:
            return error
        sharedkeyauthenticated = True

    payload = request.get_json(silent=True)
    if not isinstance(payload, dict):
        return error_response(ErrorCodes.VALIDATION_ERROR, 'No data provided')

    hostname, problem = _text_field(payload, 'hostname', 255, True)
    if problem is None:
        serialnumber, problem = _text_field(payload, 'serialnumber', 128, True)
    if problem is None:
        deviceid, problem = _text_field(payload, 'deviceid', 64, False)
    if problem is None:
        agentversion, problem = _text_field(payload, 'agentversion', 32, False)
    if problem is None:
        nonce = payload.get('enrollnonce')
        if not isinstance(nonce, str) or not _NONCE_PATTERN.match(nonce):
            problem = 'enrollnonce must be 32-128 characters of A-Z a-z 0-9 _ -'
    if problem is not None:
        return error_response(ErrorCodes.VALIDATION_ERROR, problem)

    now = _naive_utcnow()
    sourceip = client_ip(request)

    row = (EndpointEnrollRequest.query
           .filter(db.func.lower(EndpointEnrollRequest.hostname) == hostname.lower(),
                   db.func.lower(EndpointEnrollRequest.serialnumber)
                   == serialnumber.lower())
           .order_by(EndpointEnrollRequest.requestid.desc())
           .first())

    if row is None:
        row = EndpointEnrollRequest(
            hostname=hostname, serialnumber=serialnumber, enrollnonce=nonce,
            status=EndpointEnrollRequest.STATUS_PENDING, requestedat=now)
        db.session.add(row)
    else:
        samenonce = hmac.compare_digest(row.enrollnonce, nonce)
        if row.status == EndpointEnrollRequest.STATUS_APPROVED:
            row.lastseenat = now
            row.sourceip = sourceip
            if not samenonce:
                db.session.commit()
                return error_response(
                    CODE_ENROLL_NONCE_MISMATCH,
                    'This enrollment was approved for a different agent '
                    'installation', http_code=403)
            if agentauth.active_credential_for(row.matchedassetid) is not None:
                return _pending(row, REASON_ALREADY_ENROLLED, row.matchedassetid)
            if agentauth.reenroll_blocked(row.matchedassetid):
                return _pending(row, REASON_REENROLL_BLOCKED, row.matchedassetid)
            if row.matchedassetid is None or db.session.get(Asset, row.matchedassetid) is None:
                return _pending(row, REASON_UNKNOWN_HOSTNAME)
            return _issue(row, row.matchedassetid,
                          EndpointCredential.ENROLLED_APPROVED)
        if row.status == EndpointEnrollRequest.STATUS_PENDING and not samenonce:
            # First nonce wins; a second installation needs an admin.
            row.lastseenat = now
            row.sourceip = sourceip
            return _pending(row, REASON_NONCE_CHANGED, row.matchedassetid)
        if row.status == EndpointEnrollRequest.STATUS_REJECTED and samenonce:
            row.lastseenat = now
            row.sourceip = sourceip
            db.session.commit()
            return success_response(
                {'status': 'pending', 'requestid': row.requestid,
                 'reason': REASON_REJECTED}, http_code=202)
        if row.status != EndpointEnrollRequest.STATUS_PENDING:
            # Issued earlier (credential since revoked) or rejected for another
            # installation: a new enrollment cycle bound to this nonce.
            row.enrollnonce = nonce
            row.requestedat = now
            row.decidedat = None
            row.decidedby = None

    row.lastseenat = now
    row.sourceip = sourceip
    if deviceid:
        row.deviceid = deviceid
    if agentversion:
        row.agentversion = agentversion
    db.session.flush()

    Computer = _computer_model()
    matches = (Computer.query
               .filter(db.func.lower(Computer.hostname) == hostname.lower())
               .all())
    if not matches:
        return _pending(row, REASON_UNKNOWN_HOSTNAME)
    if len(matches) > 1:
        return _pending(row, REASON_DUPLICATE_HOSTNAME)

    assetid = matches[0].assetid
    asset = db.session.get(Asset, assetid)
    recorded = ((asset.serialnumber if asset is not None else None) or '').strip()
    if not recorded:
        return _pending(row, REASON_NO_SERIAL, assetid)
    if recorded.lower() != serialnumber.lower():
        return _pending(row, REASON_SERIAL_MISMATCH, assetid)
    if agentauth.active_credential_for(assetid) is not None:
        return _pending(row, REASON_ALREADY_ENROLLED, assetid)
    if agentauth.reenroll_blocked(assetid):
        return _pending(row, REASON_REENROLL_BLOCKED, assetid)
    if sharedkeyauthenticated and agentauth.auto_enroll_enabled():
        return _issue(row, assetid, EndpointCredential.ENROLLED_AUTO)
    return _pending(row, REASON_AWAITING_APPROVAL, assetid)


# ---------------------------------------------------------------------------
# Admin: credentials and requests
# ---------------------------------------------------------------------------

def _computers():
    """(assetid, hostname, serialnumber) for every active computer."""
    Computer = _computer_model()
    return (db.session.query(Computer.assetid, Computer.hostname,
                             Asset.serialnumber)
            .join(Asset, Asset.assetid == Computer.assetid)
            .filter(Asset.isactive.is_(True))
            .order_by(db.func.lower(Computer.hostname))
            .all())


def _pending_requests():
    return (EndpointEnrollRequest.query
            .filter_by(status=EndpointEnrollRequest.STATUS_PENDING)
            .all())


def _pending_index(requests):
    assetids = {r.matchedassetid for r in requests if r.matchedassetid}
    hostnames = {(r.hostname or '').lower() for r in requests}
    return assetids, hostnames


def _is_pending(assetid, hostname, index):
    assetids, hostnames = index
    return assetid in assetids or (hostname or '').lower() in hostnames


def _search_filter(rows, search, *keys):
    if not search:
        return rows
    needle = search.lower()
    return [row for row in rows
            if any(needle in str(row.get(key) or '').lower() for key in keys)]


def _page(rows):
    page, perpage = get_pagination_params()
    total = len(rows)
    start = (page - 1) * perpage
    totalpages = (total + perpage - 1) // perpage if perpage else 0
    return rows[start:start + perpage], {
        'page': page, 'perpage': perpage, 'total': total,
        'totalpages': totalpages, 'hasnext': page < totalpages,
        'hasprev': page > 1,
    }


def _settings_payload():
    return {'sharedkeyallowed': agentauth.shared_key_allowed(),
            'autoenroll': agentauth.auto_enroll_enabled()}


@endpoints_bp.route('/enrollment', methods=['GET'])
@require_permission('endpoints.fleet')
def get_enrollment():
    """Credential state of every computer, plus pending and recent requests."""
    credentials = {c.assetid: c for c in EndpointCredential.query.all()}
    pending = _pending_requests()
    index = _pending_index(pending)

    rows = []
    for assetid, hostname, serialnumber in _computers():
        credential = credentials.get(assetid)
        blocked = bool(credential is not None and credential.reenrollblocked)
        if credential is not None and credential.revokedat is None:
            state = 'enrolled'
        elif blocked:
            # Its agent keeps asking; those requests are expected, not pending work.
            state = 'revoked'
        elif _is_pending(assetid, hostname, index):
            state = 'pending'
        elif credential is not None:
            state = 'revoked'
        else:
            state = 'notenrolled'
        rows.append({
            'assetid': assetid,
            'hostname': hostname,
            'serialnumber': serialnumber,
            'state': state,
            'credentialprefix': credential.credentialprefix if credential else None,
            'issuedat': _utciso(credential.issuedat) if credential else None,
            'lastusedat': _utciso(credential.lastusedat) if credential else None,
            'enrolledvia': credential.enrolledvia if credential else None,
            'reenrollblocked': blocked,
            'reenrollblockedreason': (credential.reenrollblockedreason
                                      if blocked else None),
        })

    counts = Counter(row['state'] for row in rows)
    summary = {
        'enrolled': counts['enrolled'],
        'notenrolled': counts['notenrolled'] + counts['revoked'],
        'pending': counts['pending'],
    }

    state = (request.args.get('state') or '').strip().lower()
    if state:
        if state not in ENROLLMENT_STATES:
            return error_response(ErrorCodes.VALIDATION_ERROR,
                                  f'Unknown state filter: {state!r}')
        rows = [row for row in rows if row['state'] == state]
    rows = _search_filter(rows, (request.args.get('search') or '').strip(),
                          'hostname', 'serialnumber')
    pagerows, pagination = _page(rows)

    recent = (EndpointEnrollRequest.query
              .filter(EndpointEnrollRequest.status
                      != EndpointEnrollRequest.STATUS_PENDING)
              .order_by(EndpointEnrollRequest.lastseenat.desc())
              .limit(_RECENT_REQUESTS)
              .all())
    requests_payload = [
        r.to_dict()
        for r in sorted(pending, key=lambda r: r.lastseenat or r.requestedat,
                        reverse=True) + recent
    ]

    return success_response({
        'settings': _settings_payload(),
        'counts': summary,
        'computers': pagerows,
        'requests': requests_payload,
    }, meta={'pagination': pagination})


@endpoints_bp.route('/enrollment/requests/<int:requestid>/approve',
                    methods=['POST'])
@require_role('admin')
def approve_enroll_request(requestid):
    """Bind a pending request to the asset the admin picked."""
    row = db.session.get(EndpointEnrollRequest, requestid)
    if row is None:
        return error_response(ErrorCodes.NOT_FOUND,
                              f'No enroll request {requestid}', http_code=404)
    if row.status == EndpointEnrollRequest.STATUS_ISSUED:
        return error_response(ErrorCodes.CONFLICT,
                              'A credential was already issued for this request',
                              http_code=409)

    payload = request.get_json(silent=True) or {}
    assetid = payload.get('assetid')
    if isinstance(assetid, bool) or not isinstance(assetid, int):
        return error_response(ErrorCodes.VALIDATION_ERROR,
                              'assetid must be an integer')
    Computer = _computer_model()
    if Computer.query.filter_by(assetid=assetid).first() is None:
        return error_response(ErrorCodes.VALIDATION_ERROR,
                              f'Asset {assetid} is not a computer')
    if agentauth.active_credential_for(assetid) is not None:
        return error_response(ErrorCodes.CONFLICT,
                              'That computer is already enrolled; revoke its '
                              'credential first', http_code=409)
    if agentauth.reenroll_blocked(assetid):
        return error_response(ErrorCodes.CONFLICT,
                              'That computer is blocked from enrolling again; '
                              'allow re-enrollment first', http_code=409)

    row.status = EndpointEnrollRequest.STATUS_APPROVED
    row.matchedassetid = assetid
    row.reason = None
    row.decidedat = _naive_utcnow()
    row.decidedby = _username()
    AuditLog.log('updated', 'EndpointEnrollRequest', entityid=row.requestid,
                 entityname=row.hostname,
                 details='enroll request approved for asset {}'.format(assetid))
    db.session.commit()
    return success_response(row.to_dict(), message='Enroll request approved')


@endpoints_bp.route('/enrollment/requests/<int:requestid>/reject',
                    methods=['POST'])
@require_role('admin')
def reject_enroll_request(requestid):
    row = db.session.get(EndpointEnrollRequest, requestid)
    if row is None:
        return error_response(ErrorCodes.NOT_FOUND,
                              f'No enroll request {requestid}', http_code=404)
    if row.status == EndpointEnrollRequest.STATUS_ISSUED:
        return error_response(ErrorCodes.CONFLICT,
                              'A credential was already issued for this request; '
                              'revoke the credential instead', http_code=409)
    row.status = EndpointEnrollRequest.STATUS_REJECTED
    row.decidedat = _naive_utcnow()
    row.decidedby = _username()
    AuditLog.log('updated', 'EndpointEnrollRequest', entityid=row.requestid,
                 entityname=row.hostname, details='enroll request rejected')
    db.session.commit()
    return success_response(row.to_dict(), message='Enroll request rejected')


@endpoints_bp.route('/enrollment/assets/<int:assetid>/revoke', methods=['POST'])
@require_role('admin')
def revoke_endpoint_credential(assetid):
    """Un-enroll a PC. Its agent gets CREDENTIAL_INVALID and enrolls again.

    Body (optional) {"blockreenroll": true, "reason": "rollback"} also keeps the
    PC from enrolling again until POST .../allow-reenroll. That is the step
    before rolling a PC back to an agent that only knows the shared key: without
    it the current agent re-enrolls within seconds and the older agent is then
    refused. A PC whose credential is already revoked can still be blocked.
    """
    payload = request.get_json(silent=True) or {}
    if not isinstance(payload, dict):
        return error_response(ErrorCodes.VALIDATION_ERROR,
                              'Body must be a JSON object')
    blockreenroll = payload.get('blockreenroll', False)
    if not isinstance(blockreenroll, bool):
        return error_response(ErrorCodes.VALIDATION_ERROR,
                              'blockreenroll must be true or false')
    reason = None
    if blockreenroll:
        reason, problem = _text_field(payload, 'reason', 64, False)
        if problem is not None:
            return error_response(ErrorCodes.VALIDATION_ERROR, problem)
        reason = reason or None

    credential = agentauth.active_credential_for(assetid)
    if credential is None and blockreenroll:
        credential = EndpointCredential.query.filter_by(assetid=assetid).first()
    if credential is None:
        return error_response(ErrorCodes.NOT_FOUND,
                              'This computer has no active credential',
                              http_code=404)

    wasactive = credential.revokedat is None
    if wasactive:
        credential.revokedat = _naive_utcnow()
        credential.revokedby = _username()
    details = []
    if wasactive:
        details.append('endpoint credential {} revoked'.format(
            credential.credentialprefix))
    if blockreenroll:
        credential.reenrollblocked = True
        credential.reenrollblockedreason = reason
        details.append('re-enrollment blocked ({})'.format(reason or 'no reason'))
    AuditLog.log('deleted' if wasactive else 'updated', 'EndpointCredential',
                 entityid=assetid, entityname=credential.credentialprefix,
                 details='; '.join(details))
    db.session.commit()
    return success_response(credential.to_dict(),
                            message='Endpoint credential revoked')


@endpoints_bp.route('/enrollment/assets/<int:assetid>/allow-reenroll',
                    methods=['POST'])
@require_role('admin')
def allow_endpoint_reenroll(assetid):
    """Lift a re-enrollment block. The PC enrolls on its next attempt."""
    credential = EndpointCredential.query.filter_by(assetid=assetid).first()
    if credential is None:
        return error_response(ErrorCodes.NOT_FOUND,
                              'This computer has no credential record',
                              http_code=404)
    if not credential.reenrollblocked:
        return success_response(credential.to_dict(),
                                message='Re-enrollment was not blocked')
    previous = credential.reenrollblockedreason
    credential.reenrollblocked = False
    credential.reenrollblockedreason = None
    AuditLog.log('updated', 'EndpointCredential', entityid=assetid,
                 entityname=credential.credentialprefix,
                 changes={'reenrollblocked': {'old': True, 'new': False}},
                 details='re-enrollment allowed again (was blocked: {})'.format(
                     previous or 'no reason'))
    db.session.commit()
    return success_response(credential.to_dict(),
                            message='Re-enrollment allowed')


@endpoints_bp.route('/enrollment/settings', methods=['PUT'])
@require_role('admin')
def update_enrollment_settings():
    from shopdb.api import Setting

    payload = request.get_json(silent=True) or {}
    changes = {}
    for key, settingkey in (('sharedkeyallowed', agentauth.SHARED_KEY_SETTING),
                            ('autoenroll', agentauth.AUTO_ENROLL_SETTING)):
        if key not in payload:
            continue
        if not isinstance(payload[key], bool):
            return error_response(ErrorCodes.VALIDATION_ERROR,
                                  f'{key} must be true or false')
        changes[key] = (settingkey, payload[key])
    if not changes:
        return error_response(ErrorCodes.VALIDATION_ERROR,
                              'Send sharedkeyallowed and/or autoenroll')

    before = _settings_payload()
    for key, (settingkey, value) in changes.items():
        Setting.set(settingkey, value, valuetype='boolean',
                    category='plugin.endpoints')
    after = _settings_payload()
    AuditLog.log('updated', 'Setting', entityname='endpoint enrollment',
                 changes={key: {'old': before[key], 'new': after[key]}
                          for key in changes},
                 details='endpoint enrollment settings changed')
    db.session.commit()
    return success_response(after, message='Enrollment settings saved')


# ---------------------------------------------------------------------------
# Migration of PCs already in the fleet
# ---------------------------------------------------------------------------

def _version_below(reported, target):
    parsed = parse_agent_version(reported)
    wanted = parse_agent_version(target)
    if parsed is None or wanted is None:
        return True
    width = max(len(parsed), len(wanted))
    return (tuple(parsed) + (0,) * (width - len(parsed))
            < tuple(wanted) + (0,) * (width - len(wanted)))


def _upgrade_scriptid():
    from .agentupgradescript import UPGRADE_SCRIPT_NAME

    script = Script.query.filter_by(scriptname=UPGRADE_SCRIPT_NAME).first()
    return script.scriptid if script is not None else None


def _store_has_package():
    from . import agentstore

    current = agentstore.current_version('agent')
    if current is None:
        return agentstore.current_path('agent') is not None
    return not _version_below(current, CREDENTIAL_AGENT_VERSION)


def _latest_commands(assetids, target=None):
    """The newest run_script command per asset (optionally for one target)."""
    if not assetids:
        return {}
    newest = (db.session.query(db.func.max(EndpointCommand.endpointcommandid))
              .filter(EndpointCommand.assetid.in_(assetids),
                      EndpointCommand.action == 'run_script'))
    if target is not None:
        newest = newest.filter(EndpointCommand.target == target)
    newest = newest.group_by(EndpointCommand.assetid)
    return {command.assetid: command for command in EndpointCommand.query
            .filter(EndpointCommand.endpointcommandid.in_(newest)).all()}


def _is_tls_trust_failure(command):
    if command is None or command.status != EndpointCommand.STATUS_FAILED:
        return False
    text = '{} {}'.format(command.resultmessage or '',
                          command.resultjson or '').lower()
    return any(marker in text for marker in _TLS_TRUST_FAILURES)


def compute_readiness(assetids=None):
    """Migration state for every active computer (or just assetids).

    Returns a list of dicts ordered by hostname: assetid, hostname,
    agentversion, lastreportedat, state, blockers.
    """
    computers = _computers()
    hostnamecounts = Counter((hostname or '').lower()
                             for _assetid, hostname, _serial in computers)
    if assetids is not None:
        wanted = set(assetids)
        computers = [row for row in computers if row[0] in wanted]
    if not computers:
        return []

    ids = [row[0] for row in computers]
    credentialrows = EndpointCredential.query.filter(
        EndpointCredential.assetid.in_(ids)).all()
    active = {c.assetid for c in credentialrows if c.revokedat is None}
    reenrollblocked = {c.assetid for c in credentialrows
                       if c.reenrollblocked and c.revokedat is not None}
    index = _pending_index(_pending_requests())
    states = {s.assetid: s for s in
              EndpointState.query.filter(EndpointState.assetid.in_(ids)).all()}

    scriptid = _upgrade_scriptid()
    upgrading = set()
    if scriptid is not None:
        upgrading = {row[0] for row in db.session.query(EndpointCommand.assetid)
                     .filter(EndpointCommand.assetid.in_(ids),
                             EndpointCommand.action == 'run_script',
                             EndpointCommand.target == f'script:{scriptid}',
                             EndpointCommand.status.in_(
                                 (EndpointCommand.STATUS_PENDING,
                                  EndpointCommand.STATUS_RUNNING)))}

    lastscripts = _latest_commands(ids)

    haspackage = _store_has_package()
    minimum = minimum_agent_version_for('run_script')
    offlinecutoff = _naive_utcnow() - _OFFLINE_AFTER

    result = []
    for assetid, hostname, serialnumber in computers:
        state = states.get(assetid)
        agentversion = state.agentversion if state is not None else None
        lastreportedat = state.lastreportedat if state is not None else None
        blockers = []
        if assetid in active:
            status = 'enrolled'
        elif assetid in reenrollblocked:
            status = 'blocked'
            blockers.append(BLOCKER_REENROLL_BLOCKED)
        elif _is_pending(assetid, hostname, index):
            status = 'pending'
        elif assetid in upgrading:
            status = 'upgrading'
        elif state is None:
            status = 'notreporting'
        else:
            if lastreportedat is None or lastreportedat < offlinecutoff:
                blockers.append(BLOCKER_OFFLINE)
            if not (serialnumber or '').strip():
                blockers.append(BLOCKER_NO_SERIAL)
            if hostnamecounts[(hostname or '').lower()] > 1:
                blockers.append(BLOCKER_DUPLICATE_HOSTNAME)
            if ACTIVE_LOCK_FOR(assetid) is not None:
                blockers.append(BLOCKER_LOCKED)
            if not haspackage:
                blockers.append(BLOCKER_NO_PACKAGE)
            if minimum and not agent_meets_minimum(agentversion, minimum):
                blockers.append(BLOCKER_TOO_OLD)
            if _is_tls_trust_failure(lastscripts.get(assetid)):
                blockers.append(BLOCKER_CERT_UNTRUSTED)
            if not _version_below(agentversion, CREDENTIAL_AGENT_VERSION):
                blockers.append(BLOCKER_AWAITING_ENROLLMENT)
            if not blockers and 'run_script' not in ALLOWED_ACTIONS_FOR(
                    assetid=assetid):
                blockers.append(BLOCKER_POLICY)
            status = 'blocked' if blockers else 'ready'
        result.append({
            'assetid': assetid,
            'hostname': hostname,
            'agentversion': agentversion,
            'lastreportedat': _utciso(lastreportedat),
            'state': status,
            'blockers': blockers,
        })
    return result


def _wave_progress(wave, scriptid):
    """Live progress of one wave.

    failed counts a member only while it is still stuck: its newest agent
    upgrade (from any wave or a single push) failed or expired, it is not
    enrolled, and it does not report an agent at or above the credential
    version. A PC a later wave upgraded stops counting here.
    """
    members = wave.assetidlist
    enrolled = set()
    failed = set()
    queued = set()
    if members:
        enrolled = {c.assetid for c in EndpointCredential.query.filter(
            EndpointCredential.assetid.in_(members),
            EndpointCredential.revokedat.is_(None)).all()}
        if scriptid is not None:
            target = f'script:{scriptid}'
            queued = {row[0] for row in db.session.query(EndpointCommand.assetid)
                      .filter(EndpointCommand.assetid.in_(members),
                              EndpointCommand.action == 'run_script',
                              EndpointCommand.target == target,
                              EndpointCommand.createdat >= wave.createdat)
                      .distinct()}
            versions = dict(db.session.query(EndpointState.assetid,
                                             EndpointState.agentversion)
                            .filter(EndpointState.assetid.in_(members)).all())
            failed = {assetid for assetid, command
                      in _latest_commands(members, target).items()
                      if assetid not in enrolled
                      and command.status in (EndpointCommand.STATUS_FAILED,
                                             EndpointCommand.STATUS_EXPIRED)
                      and _version_below(versions.get(assetid),
                                         CREDENTIAL_AGENT_VERSION)}
    return {
        'waveid': wave.waveid,
        'createdat': _utciso(wave.createdat),
        'createdby': wave.createdby,
        'size': len(members),
        'queued': len(queued),
        'enrolled': len(enrolled),
        'failed': len(failed),
        'done': len(enrolled) + len(failed) >= len(members),
    }


@endpoints_bp.route('/enrollment/migration', methods=['GET'])
@require_permission('endpoints.fleet')
def get_enrollment_migration():
    """Per-computer readiness to move onto per-PC credentials, plus waves."""
    rows = compute_readiness()
    counts = Counter(row['state'] for row in rows)

    state = (request.args.get('state') or '').strip().lower()
    if state:
        if state not in MIGRATION_STATES:
            return error_response(ErrorCodes.VALIDATION_ERROR,
                                  f'Unknown state filter: {state!r}')
        rows = [row for row in rows if row['state'] == state]
    rows = _search_filter(rows, (request.args.get('search') or '').strip(),
                          'hostname', 'agentversion')
    pagerows, pagination = _page(rows)

    scriptid = _upgrade_scriptid()
    waves = (EndpointMigrationWave.query
             .order_by(EndpointMigrationWave.createdat.desc())
             .limit(_RECENT_REQUESTS)
             .all())

    return success_response({
        'target': CREDENTIAL_AGENT_VERSION,
        'counts': {name: counts[name] for name in MIGRATION_STATES},
        'computers': pagerows,
        'waves': [_wave_progress(wave, scriptid) for wave in waves],
    }, meta={'pagination': pagination})


def _response_message(response):
    try:
        body = response.get_json() or {}
    except Exception:  # noqa: BLE001 - reason text only
        return 'enqueue refused'
    error = ((body.get('data') or {}).get('error') or {})
    return error.get('message') or body.get('message') or 'enqueue refused'


@endpoints_bp.route('/enrollment/migration/wave', methods=['POST'])
@require_role('admin')
def create_migration_wave():
    """Push the agent upgrade to a batch of READY PCs.

    Body {"assetids": [...]} or {"next": n}. Readiness is re-checked per PC; the
    agent is packaged once for the whole wave, then each PC is queued through
    the same path as POST /<asset>/agentupgrade.
    """
    from .routes import prepare_agent_upgrade, queue_prepared_agent_upgrade

    payload = request.get_json(silent=True) or {}
    requested = payload.get('assetids')
    nextcount = payload.get('next')

    if requested is not None:
        if (not isinstance(requested, list) or not requested
                or not all(isinstance(v, int) and not isinstance(v, bool)
                           for v in requested)):
            return error_response(ErrorCodes.VALIDATION_ERROR,
                                  'assetids must be a non-empty list of integers')
        if len(requested) > _WAVE_MAXIMUM:
            return error_response(ErrorCodes.VALIDATION_ERROR,
                                  f'A wave holds at most {_WAVE_MAXIMUM} PCs')
        candidates = list(dict.fromkeys(requested))
    elif nextcount is not None:
        if (isinstance(nextcount, bool) or not isinstance(nextcount, int)
                or nextcount < 1 or nextcount > _WAVE_MAXIMUM):
            return error_response(ErrorCodes.VALIDATION_ERROR,
                                  f'next must be an integer from 1 to {_WAVE_MAXIMUM}')
        candidates = [row['assetid'] for row in compute_readiness()
                      if row['state'] == 'ready'][:nextcount]
    else:
        return error_response(ErrorCodes.VALIDATION_ERROR,
                              'Send assetids or next')

    readiness = {row['assetid']: row for row in compute_readiness(candidates)}
    skipped = []
    ready = []
    for assetid in candidates:
        row = readiness.get(assetid)
        if row is None:
            skipped.append({'assetid': assetid, 'reason': 'not a computer'})
        elif row['state'] != 'ready':
            reason = ', '.join(row['blockers']) if row['blockers'] else row['state']
            skipped.append({'assetid': assetid, 'reason': reason})
        else:
            ready.append(assetid)

    queued = []
    waveid = None
    if ready:
        startedat = _naive_utcnow().replace(microsecond=0)
        registration, seeded, error = prepare_agent_upgrade()
        if error is not None:
            return error
        for assetid in ready:
            response = queue_prepared_agent_upgrade(assetid, registration,
                                                    seeded, {})
            if getattr(response, 'status_code', None) == 201:
                queued.append(assetid)
            else:
                db.session.rollback()
                skipped.append({'assetid': assetid,
                                'reason': _response_message(response)})

        if queued:
            wave = EndpointMigrationWave(
                createdat=startedat, createdby=_username(),
                assetids=','.join(str(v) for v in queued))
            db.session.add(wave)
            db.session.flush()
            waveid = wave.waveid
            AuditLog.log('created', 'EndpointMigrationWave', entityid=waveid,
                         entityname='agent {}'.format(registration['agentversion']),
                         details='migration wave queued agent upgrade for {} PC(s): {}'
                                 .format(len(queued), wave.assetids))
            db.session.commit()

    return success_response(
        {'waveid': waveid, 'queued': queued, 'skipped': skipped},
        message='{} PC(s) queued'.format(len(queued)),
        http_code=201 if queued else 200)