Act on the wj architectural review: races, limits, pool and contract gaps

Findings F2, F4, F5, F6, F7, F8, F9, F10, F11 and F13 from
wj-architectural-review.md. F1/F1a (the non-sargable LOWER pattern) and F3
(the warranty sync holding a pooled connection) are being handled separately
and are untouched here.

- A collector check-in that raced another for the same PC lost everything:
  the unique index on the application fingerprint raised, nothing caught it,
  and the whole check-in rolled back, discarding state, events and hardware,
  answering the agent with form copy meant for a person. The insert now runs
  in a savepoint, reconciles what the other transaction already wrote, one row
  per savepoint so a second collision skips a row rather than the batch, and
  the check-in commits with a warning. This is the shape the delete side of
  the same function already had.
- Enrollment is deliberately unauthenticated, so it now carries a per-address
  rate limit and a cap on pending requests, and the cap is a setting an admin
  can see and change. A flood can no longer bury the approval queue or grow
  the table without bound.
- The fleet alert poll no longer formats debug strings on every request.
- BusinessUnit is on the contract surface (0.34.0), so the endpoints plugin
  stops reaching into a core table with raw SQL.
- A test walks every asset type descriptor and proves the endpoint it names
  really refuses a caller without the declared permission.
- The plugin settings routes now need a permission; they were telling anyone
  which modules a site runs and where its components live.
- The connection pool is sized explicitly with a short timeout, and the
  arithmetic against the server's connection ceiling is written next to the
  worker count instead of being left to defaults nobody chose.
- Fleet alerts carry both the correct lowercase keys and the old camelCase
  pair. Trays older than 1.7.0 match keys case-sensitively and would have gone
  quiet on every PC not yet upgraded; the old pair goes when they are.
- The dev server proxy points at localhost again, with environment variables
  for the remote-box workflow, and one developer's address is out of the
  server certificate page.

plugins/endpoints/api/bootstrap.py

@@ -28,18 +28,18 @@ import hashlib
 import json
 import re
 import secrets
-import time
 from datetime import datetime, timedelta, timezone
 
 from flask import Response, current_app, request, send_file
 
 from shopdb.api import (
-    db, cache, AuditLog, Setting, client_ip, error_response, success_response,
+    db, AuditLog, Setting, client_ip, error_response, success_response,
     ErrorCodes, require_role,
 )
 
 from ..models import EndpointBootstrapToken
 from . import siteagent
+from .ratelimit import ratelimited, rate_limit_response
 from .routes import endpoints_bp
 
 # sdbb_ + 43 chars base64url (32 random bytes, unpadded). Same shape as the
@@ -90,29 +90,20 @@ def _caller_ip():
 
 
 def _ratelimited():
-    """True when this caller is over budget for the current window."""
-    if not current_app.config.get('ENDPOINTS_BOOTSTRAP_RATELIMIT_ENABLED', True):
-        return False
-    window = current_app.config.get(
-        'ENDPOINTS_BOOTSTRAP_RATELIMIT_WINDOW_SECONDS',
-        BOOTSTRAP_RATELIMIT_WINDOW_SECONDS)
-    maxhits = current_app.config.get('ENDPOINTS_BOOTSTRAP_RATELIMIT_MAX',
-                                     BOOTSTRAP_RATELIMIT_MAX)
-    # Time bucket makes this a fixed window: the key rolls over at each window
-    # boundary, so a per-hit set() cannot turn it into a sliding window.
-    bucket = int(time.time() // window) if window > 0 else 0
-    key = 'endpointbootstrapratelimit:{0}:{1}'.format(_caller_ip(), bucket)
-    count = cache.get(key) or 0
-    if count >= maxhits:
-        return True
-    cache.set(key, count + 1, timeout=window)
-    return False
+    """True when this caller is over budget for the current window.
+
+    The implementation moved to .ratelimit so the enrollment route shares it
+    rather than carrying a second copy. Budget name, config keys and defaults
+    are unchanged, so the behaviour here is identical.
+    """
+    return ratelimited('endpointbootstrapratelimit',
+                       'ENDPOINTS_BOOTSTRAP_RATELIMIT',
+                       BOOTSTRAP_RATELIMIT_MAX,
+                       BOOTSTRAP_RATELIMIT_WINDOW_SECONDS)
 
 
 def _rate_limit_response():
-    return error_response('RATE_LIMITED',
-                          'Too many bootstrap requests. Try again later.',
-                          http_code=429)
+    return rate_limit_response('Too many bootstrap requests. Try again later.')
 
 
 def _not_found():

plugins/endpoints/api/enrollment.py

@@ -32,9 +32,9 @@ from flask import current_app, request
 from flask_jwt_extended import current_user
 
 from shopdb.api import (
-    db, Asset, AuditLog, Script, error_response, success_response, ErrorCodes,
-    require_permission, require_role, collector_authorized, collector_ingest,
-    client_ip, get_pagination_params,
+    db, Asset, AuditLog, Script, Setting, error_response, success_response,
+    ErrorCodes, require_permission, require_role, collector_authorized,
+    collector_ingest, client_ip, get_pagination_params,
 )
 
 from ..commands import (
@@ -46,12 +46,37 @@ from ..models import (
     EndpointMigrationWave, EndpointState,
 )
 from . import agentauth
+from .ratelimit import ratelimited, rate_limit_response
 from .routes import endpoints_bp
 
 # First agent that enrolls itself.
 CREDENTIAL_AGENT_VERSION = '1.20.0'
 
 CODE_ENROLL_NONCE_MISMATCH = 'ENROLL_NONCE_MISMATCH'
+CODE_ENROLL_PENDING_FULL = 'ENROLL_PENDING_FULL'
+
+# Fixed-window per-IP limiter on /enroll, which takes no auth by design and
+# writes a row for every new (hostname, serialnumber) pair it has not seen.
+# Sized for a whole shop floor behind one address, not for one PC: an agent
+# waiting for approval re-asks about once a minute, so 300 in 60 seconds covers
+# roughly 300 machines sharing a NAT address, or a re-enrollment wave across a
+# smaller room, while still turning a scripted loop into a 429 within seconds.
+# Same shape and cache as the bootstrap and login limiters (see .ratelimit).
+ENROLL_RATELIMIT_MAX = 300
+ENROLL_RATELIMIT_WINDOW_SECONDS = 60
+
+# Ceiling on rows in the pending queue. The rate limit bounds how FAST an
+# unauthenticated caller can write; this bounds how MANY rows exist at all, so a
+# slow drip from many addresses cannot grow the table without limit or bury the
+# genuine requests an admin has to find in the approval queue.
+#
+# The cap refuses only a request for a hostname and serial we have never seen.
+# An existing row - pending, approved, rejected - still updates and still hands
+# out an approved credential, so a real fleet at the cap keeps enrolling while
+# an admin drains the queue, and the refusal cannot lock anybody out of a PC
+# that is already in the workflow.
+PENDING_CAP_SETTING = 'plugin.endpoints.maxpendingenrollments'
+DEFAULT_PENDING_CAP = 500
 
 _NONCE_PATTERN = re.compile(r'^[A-Za-z0-9_-]{32,128}$')
 
@@ -213,6 +238,54 @@ def _text_field(payload, name, maximum, required):
     return value, None
 
 
+def pending_cap():
+    """How many pending enroll requests this site will hold. 0 = no cap."""
+    try:
+        value = int(Setting.get(PENDING_CAP_SETTING,
+                                default=DEFAULT_PENDING_CAP))
+    except (TypeError, ValueError):
+        return DEFAULT_PENDING_CAP
+    return max(value, 0)
+
+
+def _pending_queue_is_full():
+    """True when a NEW enroll request would push the queue past its cap.
+
+    Counted, not paged: the answer is a single number and the route only asks
+    on first contact for a hostname and serial, not on the repeat asks that
+    make up almost all the traffic.
+    """
+    cap = pending_cap()
+    if not cap:
+        return False
+    pending = (EndpointEnrollRequest.query
+               .filter(EndpointEnrollRequest.status
+                       == EndpointEnrollRequest.STATUS_PENDING)
+               .count())
+    return pending >= cap
+
+
+def _pending_queue_full_response(hostname, serialnumber):
+    """Refuse a new row, say so plainly in the log and leave an audit trail."""
+    cap = pending_cap()
+    current_app.logger.warning(
+        'endpoint enroll refused from %s: pending queue is at its cap of %d '
+        '(hostname %s, serial %s). Approve or reject requests in Settings > '
+        'Endpoints > Enrollment, or raise %s.',
+        client_ip(request), cap, hostname, serialnumber, PENDING_CAP_SETTING)
+    AuditLog.log(
+        'updated', 'EndpointEnrollRequest', entityname=hostname,
+        details='enroll refused from {0}: pending queue at cap {1} '
+                '(hostname {2}, serial {3})'.format(
+                    client_ip(request), cap, hostname, serialnumber))
+    db.session.commit()
+    return error_response(
+        CODE_ENROLL_PENDING_FULL,
+        'This site is not accepting new enrollment requests right now. '
+        'An administrator has to clear the pending queue.',
+        http_code=503)
+
+
 @endpoints_bp.route('/enroll', methods=['POST'])
 def enroll_endpoint():
     """A PC asks for its own credential.
@@ -224,7 +297,17 @@ def enroll_endpoint():
     and plugin.endpoints.autoenroll are on, may issue on a clean match. Without
     one, only a credential an admin already approved for this exact hostname,
     serial and nonce is handed out.
+
+    Unauthenticated and row-writing, so it carries both bounds an open door
+    needs: a per-IP fixed-window rate limit on how fast, and a settings-backed
+    cap on how many pending rows can exist. Neither touches a PC already in the
+    workflow - the cap refuses only a hostname and serial never seen before.
     """
+    if ratelimited('endpointenrollratelimit', 'ENDPOINTS_ENROLL_RATELIMIT',
+                   ENROLL_RATELIMIT_MAX, ENROLL_RATELIMIT_WINDOW_SECONDS):
+        return rate_limit_response(
+            'Too many enrollment requests. Try again later.')
+
     sharedkeyauthenticated = False
     presented = bool(request.headers.get('X-API-Key')
                      or request.headers.get('Authorization'))
@@ -265,6 +348,10 @@ def enroll_endpoint():
            .first())
 
     if row is None:
+        # First contact for this hostname and serial. This is the only branch
+        # that grows the table, so it is the only one the cap guards.
+        if _pending_queue_is_full():
+            return _pending_queue_full_response(hostname, serialnumber)
         row = EndpointEnrollRequest(
             hostname=hostname, serialnumber=serialnumber, enrollnonce=nonce,
             status=EndpointEnrollRequest.STATUS_PENDING, requestedat=now)
@@ -486,7 +573,8 @@ def _page(rows):
 
 def _settings_payload():
     return {'sharedkeyallowed': agentauth.shared_key_allowed(),
-            'autoenroll': agentauth.auto_enroll_enabled()}
+            'autoenroll': agentauth.auto_enroll_enabled(),
+            'maxpendingenrollments': pending_cap()}
 
 
 @endpoints_bp.route('/enrollment', methods=['GET'])
@@ -776,14 +864,25 @@ def update_enrollment_settings():
         if not isinstance(payload[key], bool):
             return error_response(ErrorCodes.VALIDATION_ERROR,
                                   f'{key} must be true or false')
-        changes[key] = (settingkey, payload[key])
+        changes[key] = (settingkey, payload[key], 'boolean')
+
+    if 'maxpendingenrollments' in payload:
+        value = payload['maxpendingenrollments']
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            return error_response(
+                ErrorCodes.VALIDATION_ERROR,
+                'maxpendingenrollments must be 0 or a positive whole number')
+        changes['maxpendingenrollments'] = (PENDING_CAP_SETTING, value,
+                                            'integer')
+
     if not changes:
-        return error_response(ErrorCodes.VALIDATION_ERROR,
-                              'Send sharedkeyallowed and/or autoenroll')
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'Send sharedkeyallowed, autoenroll and/or maxpendingenrollments')
 
     before = _settings_payload()
-    for key, (settingkey, value) in changes.items():
-        Setting.set(settingkey, value, valuetype='boolean',
+    for key, (settingkey, value, valuetype) in changes.items():
+        Setting.set(settingkey, value, valuetype=valuetype,
                     category='plugin.endpoints')
     after = _settings_payload()
     AuditLog.log('updated', 'Setting', entityname='endpoint enrollment',

plugins/endpoints/api/ratelimit.py

@@ -0,0 +1,52 @@
+"""Fixed-window per-IP rate limiting for the agent-facing routes.
+
+One implementation, several budgets. The bootstrap installer routes and the
+enrollment route are both unauthenticated by design - a PC with no agent has
+neither a credential nor a user session - so the only thing standing between
+them and a loop is a limiter. Each caller gets its own budget name so the two
+surfaces cannot spend each other's allowance.
+
+Same shape and same cache extension as the login limiter in
+shopdb.core.api.auth. Under the default SimpleCache the counter is per-process,
+so with N gunicorn workers the effective budget is N times the configured
+maximum; that is defence in depth rather than a hard quota, and a shared cache
+backend (Redis/memcached) tightens it to a true global budget.
+"""
+
+import time
+
+from flask import current_app, request
+
+from shopdb.api import cache, client_ip, error_response
+
+
+def caller_ip():
+    return client_ip(request) or 'unknown'
+
+
+def ratelimited(budget, configprefix, defaultmax, defaultwindow):
+    """True when this caller is over budget for the current window.
+
+    budget        cache key namespace, e.g. 'endpointenrollratelimit'
+    configprefix  config key stem, e.g. 'ENDPOINTS_ENROLL_RATELIMIT'; reads
+                  <prefix>_ENABLED, <prefix>_MAX and <prefix>_WINDOW_SECONDS
+    """
+    if not current_app.config.get(configprefix + '_ENABLED', True):
+        return False
+    window = current_app.config.get(configprefix + '_WINDOW_SECONDS',
+                                    defaultwindow)
+    maxhits = current_app.config.get(configprefix + '_MAX', defaultmax)
+    # Time bucket makes this a fixed window: the key rolls over at each window
+    # boundary, so a per-hit set() cannot turn it into a sliding window.
+    bucket = int(time.time() // window) if window > 0 else 0
+    key = '{0}:{1}:{2}'.format(budget, caller_ip(), bucket)
+    count = cache.get(key) or 0
+    if count >= maxhits:
+        return True
+    cache.set(key, count + 1, timeout=window)
+    return False
+
+
+def rate_limit_response(message):
+    """The standard 429 shape every limiter in the product answers with."""
+    return error_response('RATE_LIMITED', message, http_code=429)

plugins/endpoints/api/routes.py

@@ -3957,7 +3957,7 @@ def _endpoint_scope_facts(hostname):
     matchable.
     """
     from plugins.computers.models import Computer, ComputerType
-    from shopdb.api import Location, AssetType
+    from shopdb.api import BusinessUnit, Location, AssetType
 
     row = (db.session.query(Computer.hostname, AssetType.assettype,
                             ComputerType.computertype, Location.locationname,
@@ -3977,15 +3977,13 @@ def _endpoint_scope_facts(hostname):
     facts = {'hostname': hostname_value, 'assettype': assettype,
              'computertype': computertype, 'location': locationname}
 
-    # BusinessUnit is not on the plugin contract surface (shopdb.api), so resolve
-    # its name by table rather than importing the model and breaking the import
-    # contract the way the mailer import did.
+    # BusinessUnit joined the contract surface at 0.34.0 precisely so this is a
+    # primary-key get through the model rather than db.text() against a core
+    # table. A raw query here respected the letter of ADR-002 and defeated its
+    # point: hidden coupling to the core schema that no contract bump catches.
     if businessunitid is not None:
-        name = db.session.execute(
-            db.text('SELECT businessunit FROM businessunits '
-                    'WHERE businessunitid = :bid'),
-            {'bid': businessunitid}).scalar()
-        facts['businessunit'] = name
+        unit = db.session.get(BusinessUnit, businessunitid)
+        facts['businessunit'] = unit.businessunit if unit is not None else None
 
     return {k: str(v).strip().lower() for k, v in facts.items()
             if v is not None and str(v).strip()}
@@ -4099,35 +4097,39 @@ def get_fleet_alerts():
     # computer row means fleet-wide rather than a crash.
     facts = _endpoint_scope_facts(hostname) or {}
 
-    current_app.logger.debug(
-        f'[fleet_alerts] hostname={hostname} asset_id={asset_id} '
-        f'facts={facts} query_results={len(results)}')
-
     alerts = []
     for notification, ntype in results:
-        matches = _alert_targets_endpoint(notification.alertscope, facts)
-        current_app.logger.debug(
-            f'[fleet_alerts] notification_id={notification.notificationid} '
-            f'scope={notification.alertscope} matches={matches}')
-        if not matches:
+        if not _alert_targets_endpoint(notification.alertscope, facts):
             continue
 
         title = notification.notification[:100] if notification.notification else ''
         if '\n' in title:
             title = title.split('\n')[0][:100] or '(no title)'
 
-        alert_data = {
-            'notificationId': notification.notificationid,
+        alerts.append({
+            'notificationid': notification.notificationid,
             'type': ntype.typename,
-            'typeColor': ntype.typecolor or _DEFAULT_ALERT_TYPE_COLOR,
+            'typecolor': ntype.typecolor or _DEFAULT_ALERT_TYPE_COLOR,
             'title': title,
             'message': notification.notification or '',
             'timeout': notification.alerttimeout,
-        }
-        alerts.append(alert_data)
-        current_app.logger.debug(f'[fleet_alerts] added alert: {alert_data}')
+            # TRANSITION, remove once no tray older than 1.7.0 is left in the
+            # fleet: a deployed 1.6.x tray fetches this feed itself and
+            # deserializes case-sensitively, so dropping these two keys makes
+            # every alert vanish on those PCs with no error anywhere. The
+            # lowercase pair above is the contract; these two are a bridge.
+            # Any NEW field is lowercase only.
+            'notificationId': notification.notificationid,
+            'typeColor': ntype.typecolor or _DEFAULT_ALERT_TYPE_COLOR,
+        })
 
-    current_app.logger.debug(f'[fleet_alerts] returning {len(alerts)} alerts')
+    # ONE line, lazily formatted. This runs on every poll from every endpoint,
+    # so an f-string here is formatting work paid at every log level (the cost
+    # HOTFIX 1ed0c4d removed from the collector check-in); per-notification and
+    # per-alert lines multiplied it by the fleet alert count.
+    current_app.logger.debug(
+        '[fleet_alerts] hostname=%s assetid=%s candidates=%d returned=%d',
+        hostname, asset_id, len(results), len(alerts))
     return success_response(alerts)
 
 
@@ -4679,8 +4681,8 @@ def generate_server_certificate():
 
     Expects JSON body:
     {
-        "cn": "localhost",
-        "san": "DNS:localhost,IP:192.168.100.1",
+        "cn": "<hostname or IP agents connect to>",
+        "san": "DNS:<hostname>,IP:<address>",
         "days": 365
     }