Act on the wj architectural review: races, limits, pool and contract gaps

Findings F2, F4, F5, F6, F7, F8, F9, F10, F11 and F13 from
wj-architectural-review.md. F1/F1a (the non-sargable LOWER pattern) and F3
(the warranty sync holding a pooled connection) are being handled separately
and are untouched here.

- A collector check-in that raced another for the same PC lost everything:
  the unique index on the application fingerprint raised, nothing caught it,
  and the whole check-in rolled back, discarding state, events and hardware,
  answering the agent with form copy meant for a person. The insert now runs
  in a savepoint, reconciles what the other transaction already wrote, one row
  per savepoint so a second collision skips a row rather than the batch, and
  the check-in commits with a warning. This is the shape the delete side of
  the same function already had.
- Enrollment is deliberately unauthenticated, so it now carries a per-address
  rate limit and a cap on pending requests, and the cap is a setting an admin
  can see and change. A flood can no longer bury the approval queue or grow
  the table without bound.
- The fleet alert poll no longer formats debug strings on every request.
- BusinessUnit is on the contract surface (0.34.0), so the endpoints plugin
  stops reaching into a core table with raw SQL.
- A test walks every asset type descriptor and proves the endpoint it names
  really refuses a caller without the declared permission.
- The plugin settings routes now need a permission; they were telling anyone
  which modules a site runs and where its components live.
- The connection pool is sized explicitly with a short timeout, and the
  arithmetic against the server's connection ceiling is written next to the
  worker count instead of being left to defaults nobody chose.
- Fleet alerts carry both the correct lowercase keys and the old camelCase
  pair. Trays older than 1.7.0 match keys case-sensitively and would have gone
  quiet on every PC not yet upgraded; the old pair goes when they are.
- The dev server proxy points at localhost again, with environment variables
  for the remote-box workflow, and one developer's address is out of the
  server certificate page.

plugins/endpoints/api/enrollment.py

@@ -32,9 +32,9 @@ from flask import current_app, request
 from flask_jwt_extended import current_user
 
 from shopdb.api import (
-    db, Asset, AuditLog, Script, error_response, success_response, ErrorCodes,
-    require_permission, require_role, collector_authorized, collector_ingest,
-    client_ip, get_pagination_params,
+    db, Asset, AuditLog, Script, Setting, error_response, success_response,
+    ErrorCodes, require_permission, require_role, collector_authorized,
+    collector_ingest, client_ip, get_pagination_params,
 )
 
 from ..commands import (
@@ -46,12 +46,37 @@ from ..models import (
     EndpointMigrationWave, EndpointState,
 )
 from . import agentauth
+from .ratelimit import ratelimited, rate_limit_response
 from .routes import endpoints_bp
 
 # First agent that enrolls itself.
 CREDENTIAL_AGENT_VERSION = '1.20.0'
 
 CODE_ENROLL_NONCE_MISMATCH = 'ENROLL_NONCE_MISMATCH'
+CODE_ENROLL_PENDING_FULL = 'ENROLL_PENDING_FULL'
+
+# Fixed-window per-IP limiter on /enroll, which takes no auth by design and
+# writes a row for every new (hostname, serialnumber) pair it has not seen.
+# Sized for a whole shop floor behind one address, not for one PC: an agent
+# waiting for approval re-asks about once a minute, so 300 in 60 seconds covers
+# roughly 300 machines sharing a NAT address, or a re-enrollment wave across a
+# smaller room, while still turning a scripted loop into a 429 within seconds.
+# Same shape and cache as the bootstrap and login limiters (see .ratelimit).
+ENROLL_RATELIMIT_MAX = 300
+ENROLL_RATELIMIT_WINDOW_SECONDS = 60
+
+# Ceiling on rows in the pending queue. The rate limit bounds how FAST an
+# unauthenticated caller can write; this bounds how MANY rows exist at all, so a
+# slow drip from many addresses cannot grow the table without limit or bury the
+# genuine requests an admin has to find in the approval queue.
+#
+# The cap refuses only a request for a hostname and serial we have never seen.
+# An existing row - pending, approved, rejected - still updates and still hands
+# out an approved credential, so a real fleet at the cap keeps enrolling while
+# an admin drains the queue, and the refusal cannot lock anybody out of a PC
+# that is already in the workflow.
+PENDING_CAP_SETTING = 'plugin.endpoints.maxpendingenrollments'
+DEFAULT_PENDING_CAP = 500
 
 _NONCE_PATTERN = re.compile(r'^[A-Za-z0-9_-]{32,128}$')
 
@@ -213,6 +238,54 @@ def _text_field(payload, name, maximum, required):
     return value, None
 
 
+def pending_cap():
+    """How many pending enroll requests this site will hold. 0 = no cap."""
+    try:
+        value = int(Setting.get(PENDING_CAP_SETTING,
+                                default=DEFAULT_PENDING_CAP))
+    except (TypeError, ValueError):
+        return DEFAULT_PENDING_CAP
+    return max(value, 0)
+
+
+def _pending_queue_is_full():
+    """True when a NEW enroll request would push the queue past its cap.
+
+    Counted, not paged: the answer is a single number and the route only asks
+    on first contact for a hostname and serial, not on the repeat asks that
+    make up almost all the traffic.
+    """
+    cap = pending_cap()
+    if not cap:
+        return False
+    pending = (EndpointEnrollRequest.query
+               .filter(EndpointEnrollRequest.status
+                       == EndpointEnrollRequest.STATUS_PENDING)
+               .count())
+    return pending >= cap
+
+
+def _pending_queue_full_response(hostname, serialnumber):
+    """Refuse a new row, say so plainly in the log and leave an audit trail."""
+    cap = pending_cap()
+    current_app.logger.warning(
+        'endpoint enroll refused from %s: pending queue is at its cap of %d '
+        '(hostname %s, serial %s). Approve or reject requests in Settings > '
+        'Endpoints > Enrollment, or raise %s.',
+        client_ip(request), cap, hostname, serialnumber, PENDING_CAP_SETTING)
+    AuditLog.log(
+        'updated', 'EndpointEnrollRequest', entityname=hostname,
+        details='enroll refused from {0}: pending queue at cap {1} '
+                '(hostname {2}, serial {3})'.format(
+                    client_ip(request), cap, hostname, serialnumber))
+    db.session.commit()
+    return error_response(
+        CODE_ENROLL_PENDING_FULL,
+        'This site is not accepting new enrollment requests right now. '
+        'An administrator has to clear the pending queue.',
+        http_code=503)
+
+
 @endpoints_bp.route('/enroll', methods=['POST'])
 def enroll_endpoint():
     """A PC asks for its own credential.
@@ -224,7 +297,17 @@ def enroll_endpoint():
     and plugin.endpoints.autoenroll are on, may issue on a clean match. Without
     one, only a credential an admin already approved for this exact hostname,
     serial and nonce is handed out.
+
+    Unauthenticated and row-writing, so it carries both bounds an open door
+    needs: a per-IP fixed-window rate limit on how fast, and a settings-backed
+    cap on how many pending rows can exist. Neither touches a PC already in the
+    workflow - the cap refuses only a hostname and serial never seen before.
     """
+    if ratelimited('endpointenrollratelimit', 'ENDPOINTS_ENROLL_RATELIMIT',
+                   ENROLL_RATELIMIT_MAX, ENROLL_RATELIMIT_WINDOW_SECONDS):
+        return rate_limit_response(
+            'Too many enrollment requests. Try again later.')
+
     sharedkeyauthenticated = False
     presented = bool(request.headers.get('X-API-Key')
                      or request.headers.get('Authorization'))
@@ -265,6 +348,10 @@ def enroll_endpoint():
            .first())
 
     if row is None:
+        # First contact for this hostname and serial. This is the only branch
+        # that grows the table, so it is the only one the cap guards.
+        if _pending_queue_is_full():
+            return _pending_queue_full_response(hostname, serialnumber)
         row = EndpointEnrollRequest(
             hostname=hostname, serialnumber=serialnumber, enrollnonce=nonce,
             status=EndpointEnrollRequest.STATUS_PENDING, requestedat=now)
@@ -486,7 +573,8 @@ def _page(rows):
 
 def _settings_payload():
     return {'sharedkeyallowed': agentauth.shared_key_allowed(),
-            'autoenroll': agentauth.auto_enroll_enabled()}
+            'autoenroll': agentauth.auto_enroll_enabled(),
+            'maxpendingenrollments': pending_cap()}
 
 
 @endpoints_bp.route('/enrollment', methods=['GET'])
@@ -776,14 +864,25 @@ def update_enrollment_settings():
         if not isinstance(payload[key], bool):
             return error_response(ErrorCodes.VALIDATION_ERROR,
                                   f'{key} must be true or false')
-        changes[key] = (settingkey, payload[key])
+        changes[key] = (settingkey, payload[key], 'boolean')
+
+    if 'maxpendingenrollments' in payload:
+        value = payload['maxpendingenrollments']
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            return error_response(
+                ErrorCodes.VALIDATION_ERROR,
+                'maxpendingenrollments must be 0 or a positive whole number')
+        changes['maxpendingenrollments'] = (PENDING_CAP_SETTING, value,
+                                            'integer')
+
     if not changes:
-        return error_response(ErrorCodes.VALIDATION_ERROR,
-                              'Send sharedkeyallowed and/or autoenroll')
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'Send sharedkeyallowed, autoenroll and/or maxpendingenrollments')
 
     before = _settings_payload()
-    for key, (settingkey, value) in changes.items():
-        Setting.set(settingkey, value, valuetype='boolean',
+    for key, (settingkey, value, valuetype) in changes.items():
+        Setting.set(settingkey, value, valuetype=valuetype,
                     category='plugin.endpoints')
     after = _settings_payload()
     AuditLog.log('updated', 'Setting', entityname='endpoint enrollment',