Act on the wj architectural review: races, limits, pool and contract gaps

Findings F2, F4, F5, F6, F7, F8, F9, F10, F11 and F13 from
wj-architectural-review.md. F1/F1a (the non-sargable LOWER pattern) and F3
(the warranty sync holding a pooled connection) are being handled separately
and are untouched here.

- A collector check-in that raced another for the same PC lost everything:
  the unique index on the application fingerprint raised, nothing caught it,
  and the whole check-in rolled back, discarding state, events and hardware,
  answering the agent with form copy meant for a person. The insert now runs
  in a savepoint, reconciles what the other transaction already wrote, one row
  per savepoint so a second collision skips a row rather than the batch, and
  the check-in commits with a warning. This is the shape the delete side of
  the same function already had.
- Enrollment is deliberately unauthenticated, so it now carries a per-address
  rate limit and a cap on pending requests, and the cap is a setting an admin
  can see and change. A flood can no longer bury the approval queue or grow
  the table without bound.
- The fleet alert poll no longer formats debug strings on every request.
- BusinessUnit is on the contract surface (0.34.0), so the endpoints plugin
  stops reaching into a core table with raw SQL.
- A test walks every asset type descriptor and proves the endpoint it names
  really refuses a caller without the declared permission.
- The plugin settings routes now need a permission; they were telling anyone
  which modules a site runs and where its components live.
- The connection pool is sized explicitly with a short timeout, and the
  arithmetic against the server's connection ceiling is written next to the
  worker count instead of being left to defaults nobody chose.
- Fleet alerts carry both the correct lowercase keys and the old camelCase
  pair. Trays older than 1.7.0 match keys case-sensitively and would have gone
  quiet on every PC not yet upgraded; the old pair goes when they are.
- The dev server proxy points at localhost again, with environment variables
  for the remote-box workflow, and one developer's address is out of the
  server certificate page.

.env.example

@@ -28,6 +28,19 @@ JWT_SECRET_KEY=change-this-to-another-secure-random-string
 # utf8mb4 too (see the DEPLOY wiki page). Both must be utf8mb4 to match the schema.
 DATABASE_URL=mysql+pymysql://shopdb:CHANGE_ME@db:3306/shopdb_flask?charset=utf8mb4
 
+# ---- Database connection pool (optional, defaults shown) ----
+# Per WORKER process, so the real ceiling is workers x (size + overflow). See
+# the arithmetic beside --workers in docker-compose.yml before raising either:
+# at the shipped 4 workers this is 4 x (10 + 10) = 80 against the mysql:8.4
+# default max_connections of 151. More workers needs a smaller pool here or a
+# higher --max-connections on the db service.
+# DB_POOL_TIMEOUT is the wait for a free connection. 5s fails fast and
+# legibly; the SQLAlchemy default of 30s turns pool exhaustion into what looks
+# like a dead database.
+DB_POOL_SIZE=10
+DB_MAX_OVERFLOW=10
+DB_POOL_TIMEOUT=5
+
 # ---- CORS (required, no wildcards in production) ----
 
 # Comma-separated list of explicit origins permitted to call the API.

CONTRIBUTING.md

@@ -156,6 +156,43 @@ When creating a new plugin:
 
 See the PLUGIN-QUICKSTART wiki page (when published) for the 30-minute walkthrough.
 
+#### A plugin must not hold a DB session across external I/O
+
+Rule: finish with the database before you call anything outside the process -
+an HTTP API (Dell warranty, ServiceNow, Zabbix, Azure/Intune), a subprocess, a
+file on a network share - and open a fresh session afterwards to write the
+result.
+
+Why it is a rule and not a preference. The app runs gevent workers, so a
+request that blocks on a network call does not block its worker, and the call
+can take minutes without anything complaining. What it DOES hold for those
+minutes is a pooled database connection. Each worker owns a pool of
+`DB_POOL_SIZE + DB_MAX_OVERFLOW` connections (see `shopdb/config.py` and the
+arithmetic beside `--workers` in `docker-compose.yml`), and hundreds of
+greenlets can enter the same slow path at once. Drain the pool and every other
+request in that worker blocks on checkout and then fails - the whole site,
+every plugin, while the CPU sits idle and MySQL reports itself healthy. That is
+the worst diagnostic signature there is.
+
+The shape to write:
+
+```python
+# Read what you need, then let the session go.
+servicetags = [row.servicetag for row in query.all()]
+db.session.remove()
+
+results = provider.bulk_lookup(servicetags)   # slow, no connection held
+
+for servicetag, result in results.items():    # new session, short write
+    ...
+db.session.commit()
+```
+
+Rules of thumb: no `requests`, `urllib`, `subprocess` or network-share access
+between a query and its `commit()`; cap every external call with a timeout; and
+if a sync is inherently long, chunk it so each chunk is its own
+read / call / write cycle.
+
 ## Testing
 
 Run tests with:

agent/ShopDbEndpointAgent.Core/FleetAlertClient.cs

@@ -99,9 +99,9 @@ public sealed class FleetAlert
     private string title = "";
     private string message = "";
 
-    [JsonPropertyName("notificationId")] public int NotificationId { get; set; }
+    [JsonPropertyName("notificationid")] public int NotificationId { get; set; }
     [JsonPropertyName("type")] public string TypeName { get => typeName; set => typeName = value ?? ""; }
-    [JsonPropertyName("typeColor")] public string TypeColor { get => typeColor; set => typeColor = value ?? ""; }
+    [JsonPropertyName("typecolor")] public string TypeColor { get => typeColor; set => typeColor = value ?? ""; }
     [JsonPropertyName("title")] public string Title { get => title; set => title = value ?? ""; }
     [JsonPropertyName("message")] public string Message { get => message; set => message = value ?? ""; }
     [JsonPropertyName("timeout")] public int? Timeout { get; set; }
@@ -126,7 +126,7 @@ public sealed class FleetAlertList : List<FleetAlert>
 }
 
 // Deserializes the data array one element at a time. An element that fails
-// to bind, or has no usable notificationId, is recorded and skipped.
+// to bind, or has no usable notificationid, is recorded and skipped.
 public sealed class TolerantFleetAlertListConverter : JsonConverter<List<FleetAlert>>
 {
     public override List<FleetAlert>? Read(
@@ -146,7 +146,7 @@ public sealed class TolerantFleetAlertListConverter : JsonConverter<List<FleetAl
             {
                 var alert = element.RootElement.Deserialize<FleetAlert>(options);
                 if (alert is null || alert.NotificationId <= 0)
-                    list.Skipped.Add($"element {index}: missing notificationId");
+                    list.Skipped.Add($"element {index}: missing notificationid");
                 else
                     list.Add(alert);
             }

docker-compose.yml

@@ -167,6 +167,31 @@ services:
       # connections, is the binding resource.
       - --worker-class
       - gevent
+      # DATABASE CONNECTION ARITHMETIC. Read this before changing the number.
+      #
+      #   workers x (pool_size + max_overflow) + migrate + admin  vs  MySQL
+      #   max_connections
+      #
+      # Each worker process owns its own SQLAlchemy pool, so the ceiling is per
+      # worker multiplied by the worker count. With the defaults in
+      # shopdb/config.py (DB_POOL_SIZE 10, DB_MAX_OVERFLOW 10):
+      #
+      #   4 x (10 + 10) = 80
+      #   + the migrate service (a handful, and it has exited before api
+      #     starts, so it only overlaps during a manual upgrade)
+      #   + whatever a human has open (mysqldump, a client on the host)
+      #   = about 90 against the mysql:8.4 default max_connections of 151.
+      #
+      # The `command:` on the db service above sets charset and collation only,
+      # so that 151 is the server default and nothing else in this repo relates
+      # the two numbers. Raise --workers to 8 and the ceiling is 160 before any
+      # overhead, which is ABOVE 151: MySQL then refuses connections, and the
+      # failure shows up as errors from the database rather than anything in the
+      # app. So change one of these and change the others deliberately:
+      # more workers means either a smaller pool or --max-connections on db.
+      #
+      # pool_timeout is 5s, not the 30s default, so exhaustion fails fast and
+      # legibly instead of hanging every request for half a minute.
       - --workers
       - "4"
       # Per-worker greenlet ceiling. 4 x 2000 is far above any realistic fleet;

docs/deployment/LOCAL-DEV-SETUP.md

@@ -189,6 +189,16 @@ npm run dev
 
 Frontend dev server runs on http://localhost:5173 and proxies API requests to Flask on port 5001.
 
+To point the proxy at a backend on another machine, set `VITE_DEV_API_TARGET`
+in your shell instead of editing `vite.config.js`:
+
+```bash
+VITE_DEV_API_TARGET=https://10.0.0.5:5001 npm run dev
+```
+
+Certificate verification is off for an https target because those boxes
+normally use a self-signed cert; set `VITE_DEV_SECURE=true` to enforce it.
+
 For production build (served by Flask):
 ```bash
 cd frontend

frontend/src/views/settings/ModelCatalogHub.vue

@@ -48,7 +48,9 @@
 <script setup>
 import { ref, onMounted } from 'vue'
 import { useRoute, useRouter } from 'vue-router'
-import axios from 'axios'
+// The shared api instance, not raw axios: this route is gated on settings.view
+// now, and only this client attaches the bearer token and the mount-path base.
+import api from '@/api'
 
 const route = useRoute()
 const router = useRouter()
@@ -57,7 +59,7 @@ const pluginTabs = ref([])
 onMounted(async () => {
   // Fetch plugin-contributed settings routes
   try {
-    const response = await axios.get('/api/plugins/settings-routes')
+    const response = await api.get('/plugins/settings-routes')
     if (response.data && response.data.data) {
       // Filter for model-catalog hub routes
       pluginTabs.value = response.data.data.routes.filter(r => r.hub === 'model-catalog')

frontend/vite.config.js

@@ -2,6 +2,21 @@ import { defineConfig } from 'vite'
 import vue from '@vitejs/plugin-vue'
 import path from 'path'
 
+// Dev proxy target for /api and /static. Default is the local Flask dev server.
+// To develop against a backend on another box, set it per shell, never here:
+//   VITE_DEV_API_TARGET=https://10.0.0.5:5001 npm run dev
+// Certificates on such a box are usually self-signed, so verification is off by
+// default; VITE_DEV_SECURE=true turns it back on for an https target.
+const devApiTarget = process.env.VITE_DEV_API_TARGET || 'http://localhost:5001'
+const devApiSecure = devApiTarget.startsWith('https:')
+  && process.env.VITE_DEV_SECURE === 'true'
+
+const devProxy = {
+  target: devApiTarget,
+  changeOrigin: true,
+  secure: devApiSecure
+}
+
 export default defineConfig({
   // Mount path of the built app. Default '/' (own IIS site / dev). Set
   // VITE_BASE_PATH='/ops/' (trailing slash) to build for a subpath mount under
@@ -14,19 +29,12 @@ export default defineConfig({
     }
   },
   server: {
+    // all interfaces, so a VM or phone on the LAN can load the dev server
     host: true,
     port: 5173,
     proxy: {
-      '/api': {
-        target: 'https://192.168.100.1:5001',
-        changeOrigin: true,
-        secure: false
-      },
-      '/static': {
-        target: 'https://192.168.100.1:5001',
-        changeOrigin: true,
-        secure: false
-      }
+      '/api': { ...devProxy },
+      '/static': { ...devProxy }
     }
   }
 })

plugins/endpoints/AGENT-CONTRACT.md

@@ -872,3 +872,33 @@ The generated script:
   route and adding one would hand an anonymous caller a way to probe the fleet.
 
 Exit code 0 on install success even when the PC is still pending.
+
+### 9.9 Fleet alert feed
+
+`GET /api/endpoints/alerts?hostname=<host>` returns the alerts that should be
+live on that PC right now (see the AUTHORITATIVE rule above). One object per
+alert:
+
+| Key | Type | Meaning |
+|---|---|---|
+| `notificationid` | int | the notification, and the id posted back to `/alerts/<id>/acknowledge` |
+| `type` | string | notification type name |
+| `typecolor` | string | CSS colour the tray paints the toast with |
+| `title` | string | first line of the message, max 100 chars |
+| `message` | string | full message |
+| `timeout` | int or null | seconds the toast stays on screen |
+
+Response keys match the column names, per the naming convention. Two extra keys
+ride along for now:
+
+| Legacy key | Same value as | Remove when |
+|---|---|---|
+| `notificationId` | `notificationid` | no tray older than 1.7.0 is left in the fleet |
+| `typeColor` | `typecolor` | no tray older than 1.7.0 is left in the fleet |
+
+A tray at 1.6.x fetches this feed itself and deserializes case-sensitively, so
+serving only the lowercase keys makes every alert vanish on those machines with
+no error raised anywhere. Both spellings therefore go down with identical
+values until the fleet is upgraded, and the camelCase pair is deleted in the
+same release that drops support for trays below 1.7.0. Any NEW field on this
+object is lowercase only and never gets a camelCase twin.

plugins/endpoints/api/bootstrap.py

@@ -28,18 +28,18 @@ import hashlib
 import json
 import re
 import secrets
-import time
 from datetime import datetime, timedelta, timezone
 
 from flask import Response, current_app, request, send_file
 
 from shopdb.api import (
-    db, cache, AuditLog, Setting, client_ip, error_response, success_response,
+    db, AuditLog, Setting, client_ip, error_response, success_response,
     ErrorCodes, require_role,
 )
 
 from ..models import EndpointBootstrapToken
 from . import siteagent
+from .ratelimit import ratelimited, rate_limit_response
 from .routes import endpoints_bp
 
 # sdbb_ + 43 chars base64url (32 random bytes, unpadded). Same shape as the
@@ -90,29 +90,20 @@ def _caller_ip():
 
 
 def _ratelimited():
-    """True when this caller is over budget for the current window."""
-    if not current_app.config.get('ENDPOINTS_BOOTSTRAP_RATELIMIT_ENABLED', True):
-        return False
-    window = current_app.config.get(
-        'ENDPOINTS_BOOTSTRAP_RATELIMIT_WINDOW_SECONDS',
+    """True when this caller is over budget for the current window.
+
+    The implementation moved to .ratelimit so the enrollment route shares it
+    rather than carrying a second copy. Budget name, config keys and defaults
+    are unchanged, so the behaviour here is identical.
+    """
+    return ratelimited('endpointbootstrapratelimit',
+                       'ENDPOINTS_BOOTSTRAP_RATELIMIT',
+                       BOOTSTRAP_RATELIMIT_MAX,
                        BOOTSTRAP_RATELIMIT_WINDOW_SECONDS)
-    maxhits = current_app.config.get('ENDPOINTS_BOOTSTRAP_RATELIMIT_MAX',
-                                     BOOTSTRAP_RATELIMIT_MAX)
-    # Time bucket makes this a fixed window: the key rolls over at each window
-    # boundary, so a per-hit set() cannot turn it into a sliding window.
-    bucket = int(time.time() // window) if window > 0 else 0
-    key = 'endpointbootstrapratelimit:{0}:{1}'.format(_caller_ip(), bucket)
-    count = cache.get(key) or 0
-    if count >= maxhits:
-        return True
-    cache.set(key, count + 1, timeout=window)
-    return False
 
 
 def _rate_limit_response():
-    return error_response('RATE_LIMITED',
-                          'Too many bootstrap requests. Try again later.',
-                          http_code=429)
+    return rate_limit_response('Too many bootstrap requests. Try again later.')
 
 
 def _not_found():

plugins/endpoints/api/enrollment.py

@@ -32,9 +32,9 @@ from flask import current_app, request
 from flask_jwt_extended import current_user
 
 from shopdb.api import (
-    db, Asset, AuditLog, Script, error_response, success_response, ErrorCodes,
-    require_permission, require_role, collector_authorized, collector_ingest,
-    client_ip, get_pagination_params,
+    db, Asset, AuditLog, Script, Setting, error_response, success_response,
+    ErrorCodes, require_permission, require_role, collector_authorized,
+    collector_ingest, client_ip, get_pagination_params,
 )
 
 from ..commands import (
@@ -46,12 +46,37 @@ from ..models import (
     EndpointMigrationWave, EndpointState,
 )
 from . import agentauth
+from .ratelimit import ratelimited, rate_limit_response
 from .routes import endpoints_bp
 
 # First agent that enrolls itself.
 CREDENTIAL_AGENT_VERSION = '1.20.0'
 
 CODE_ENROLL_NONCE_MISMATCH = 'ENROLL_NONCE_MISMATCH'
+CODE_ENROLL_PENDING_FULL = 'ENROLL_PENDING_FULL'
+
+# Fixed-window per-IP limiter on /enroll, which takes no auth by design and
+# writes a row for every new (hostname, serialnumber) pair it has not seen.
+# Sized for a whole shop floor behind one address, not for one PC: an agent
+# waiting for approval re-asks about once a minute, so 300 in 60 seconds covers
+# roughly 300 machines sharing a NAT address, or a re-enrollment wave across a
+# smaller room, while still turning a scripted loop into a 429 within seconds.
+# Same shape and cache as the bootstrap and login limiters (see .ratelimit).
+ENROLL_RATELIMIT_MAX = 300
+ENROLL_RATELIMIT_WINDOW_SECONDS = 60
+
+# Ceiling on rows in the pending queue. The rate limit bounds how FAST an
+# unauthenticated caller can write; this bounds how MANY rows exist at all, so a
+# slow drip from many addresses cannot grow the table without limit or bury the
+# genuine requests an admin has to find in the approval queue.
+#
+# The cap refuses only a request for a hostname and serial we have never seen.
+# An existing row - pending, approved, rejected - still updates and still hands
+# out an approved credential, so a real fleet at the cap keeps enrolling while
+# an admin drains the queue, and the refusal cannot lock anybody out of a PC
+# that is already in the workflow.
+PENDING_CAP_SETTING = 'plugin.endpoints.maxpendingenrollments'
+DEFAULT_PENDING_CAP = 500
 
 _NONCE_PATTERN = re.compile(r'^[A-Za-z0-9_-]{32,128}$')
 
@@ -213,6 +238,54 @@ def _text_field(payload, name, maximum, required):
     return value, None
 
 
+def pending_cap():
+    """How many pending enroll requests this site will hold. 0 = no cap."""
+    try:
+        value = int(Setting.get(PENDING_CAP_SETTING,
+                                default=DEFAULT_PENDING_CAP))
+    except (TypeError, ValueError):
+        return DEFAULT_PENDING_CAP
+    return max(value, 0)
+
+
+def _pending_queue_is_full():
+    """True when a NEW enroll request would push the queue past its cap.
+
+    Counted, not paged: the answer is a single number and the route only asks
+    on first contact for a hostname and serial, not on the repeat asks that
+    make up almost all the traffic.
+    """
+    cap = pending_cap()
+    if not cap:
+        return False
+    pending = (EndpointEnrollRequest.query
+               .filter(EndpointEnrollRequest.status
+                       == EndpointEnrollRequest.STATUS_PENDING)
+               .count())
+    return pending >= cap
+
+
+def _pending_queue_full_response(hostname, serialnumber):
+    """Refuse a new row, say so plainly in the log and leave an audit trail."""
+    cap = pending_cap()
+    current_app.logger.warning(
+        'endpoint enroll refused from %s: pending queue is at its cap of %d '
+        '(hostname %s, serial %s). Approve or reject requests in Settings > '
+        'Endpoints > Enrollment, or raise %s.',
+        client_ip(request), cap, hostname, serialnumber, PENDING_CAP_SETTING)
+    AuditLog.log(
+        'updated', 'EndpointEnrollRequest', entityname=hostname,
+        details='enroll refused from {0}: pending queue at cap {1} '
+                '(hostname {2}, serial {3})'.format(
+                    client_ip(request), cap, hostname, serialnumber))
+    db.session.commit()
+    return error_response(
+        CODE_ENROLL_PENDING_FULL,
+        'This site is not accepting new enrollment requests right now. '
+        'An administrator has to clear the pending queue.',
+        http_code=503)
+
+
 @endpoints_bp.route('/enroll', methods=['POST'])
 def enroll_endpoint():
     """A PC asks for its own credential.
@@ -224,7 +297,17 @@ def enroll_endpoint():
     and plugin.endpoints.autoenroll are on, may issue on a clean match. Without
     one, only a credential an admin already approved for this exact hostname,
     serial and nonce is handed out.
+
+    Unauthenticated and row-writing, so it carries both bounds an open door
+    needs: a per-IP fixed-window rate limit on how fast, and a settings-backed
+    cap on how many pending rows can exist. Neither touches a PC already in the
+    workflow - the cap refuses only a hostname and serial never seen before.
     """
+    if ratelimited('endpointenrollratelimit', 'ENDPOINTS_ENROLL_RATELIMIT',
+                   ENROLL_RATELIMIT_MAX, ENROLL_RATELIMIT_WINDOW_SECONDS):
+        return rate_limit_response(
+            'Too many enrollment requests. Try again later.')
+
     sharedkeyauthenticated = False
     presented = bool(request.headers.get('X-API-Key')
                      or request.headers.get('Authorization'))
@@ -265,6 +348,10 @@ def enroll_endpoint():
            .first())
 
     if row is None:
+        # First contact for this hostname and serial. This is the only branch
+        # that grows the table, so it is the only one the cap guards.
+        if _pending_queue_is_full():
+            return _pending_queue_full_response(hostname, serialnumber)
         row = EndpointEnrollRequest(
             hostname=hostname, serialnumber=serialnumber, enrollnonce=nonce,
             status=EndpointEnrollRequest.STATUS_PENDING, requestedat=now)
@@ -486,7 +573,8 @@ def _page(rows):
 
 def _settings_payload():
     return {'sharedkeyallowed': agentauth.shared_key_allowed(),
-            'autoenroll': agentauth.auto_enroll_enabled()}
+            'autoenroll': agentauth.auto_enroll_enabled(),
+            'maxpendingenrollments': pending_cap()}
 
 
 @endpoints_bp.route('/enrollment', methods=['GET'])
@@ -776,14 +864,25 @@ def update_enrollment_settings():
         if not isinstance(payload[key], bool):
             return error_response(ErrorCodes.VALIDATION_ERROR,
                                   f'{key} must be true or false')
-        changes[key] = (settingkey, payload[key])
+        changes[key] = (settingkey, payload[key], 'boolean')
+
+    if 'maxpendingenrollments' in payload:
+        value = payload['maxpendingenrollments']
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            return error_response(
+                ErrorCodes.VALIDATION_ERROR,
+                'maxpendingenrollments must be 0 or a positive whole number')
+        changes['maxpendingenrollments'] = (PENDING_CAP_SETTING, value,
+                                            'integer')
+
     if not changes:
-        return error_response(ErrorCodes.VALIDATION_ERROR,
-                              'Send sharedkeyallowed and/or autoenroll')
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'Send sharedkeyallowed, autoenroll and/or maxpendingenrollments')
 
     before = _settings_payload()
-    for key, (settingkey, value) in changes.items():
-        Setting.set(settingkey, value, valuetype='boolean',
+    for key, (settingkey, value, valuetype) in changes.items():
+        Setting.set(settingkey, value, valuetype=valuetype,
                     category='plugin.endpoints')
     after = _settings_payload()
     AuditLog.log('updated', 'Setting', entityname='endpoint enrollment',

plugins/endpoints/api/ratelimit.py

@@ -0,0 +1,52 @@
+"""Fixed-window per-IP rate limiting for the agent-facing routes.
+
+One implementation, several budgets. The bootstrap installer routes and the
+enrollment route are both unauthenticated by design - a PC with no agent has
+neither a credential nor a user session - so the only thing standing between
+them and a loop is a limiter. Each caller gets its own budget name so the two
+surfaces cannot spend each other's allowance.
+
+Same shape and same cache extension as the login limiter in
+shopdb.core.api.auth. Under the default SimpleCache the counter is per-process,
+so with N gunicorn workers the effective budget is N times the configured
+maximum; that is defence in depth rather than a hard quota, and a shared cache
+backend (Redis/memcached) tightens it to a true global budget.
+"""
+
+import time
+
+from flask import current_app, request
+
+from shopdb.api import cache, client_ip, error_response
+
+
+def caller_ip():
+    return client_ip(request) or 'unknown'
+
+
+def ratelimited(budget, configprefix, defaultmax, defaultwindow):
+    """True when this caller is over budget for the current window.
+
+    budget        cache key namespace, e.g. 'endpointenrollratelimit'
+    configprefix  config key stem, e.g. 'ENDPOINTS_ENROLL_RATELIMIT'; reads
+                  <prefix>_ENABLED, <prefix>_MAX and <prefix>_WINDOW_SECONDS
+    """
+    if not current_app.config.get(configprefix + '_ENABLED', True):
+        return False
+    window = current_app.config.get(configprefix + '_WINDOW_SECONDS',
+                                    defaultwindow)
+    maxhits = current_app.config.get(configprefix + '_MAX', defaultmax)
+    # Time bucket makes this a fixed window: the key rolls over at each window
+    # boundary, so a per-hit set() cannot turn it into a sliding window.
+    bucket = int(time.time() // window) if window > 0 else 0
+    key = '{0}:{1}:{2}'.format(budget, caller_ip(), bucket)
+    count = cache.get(key) or 0
+    if count >= maxhits:
+        return True
+    cache.set(key, count + 1, timeout=window)
+    return False
+
+
+def rate_limit_response(message):
+    """The standard 429 shape every limiter in the product answers with."""
+    return error_response('RATE_LIMITED', message, http_code=429)

plugins/endpoints/api/routes.py

@@ -3957,7 +3957,7 @@ def _endpoint_scope_facts(hostname):
     matchable.
     """
     from plugins.computers.models import Computer, ComputerType
-    from shopdb.api import Location, AssetType
+    from shopdb.api import BusinessUnit, Location, AssetType
 
     row = (db.session.query(Computer.hostname, AssetType.assettype,
                             ComputerType.computertype, Location.locationname,
@@ -3977,15 +3977,13 @@ def _endpoint_scope_facts(hostname):
     facts = {'hostname': hostname_value, 'assettype': assettype,
              'computertype': computertype, 'location': locationname}
 
-    # BusinessUnit is not on the plugin contract surface (shopdb.api), so resolve
-    # its name by table rather than importing the model and breaking the import
-    # contract the way the mailer import did.
+    # BusinessUnit joined the contract surface at 0.34.0 precisely so this is a
+    # primary-key get through the model rather than db.text() against a core
+    # table. A raw query here respected the letter of ADR-002 and defeated its
+    # point: hidden coupling to the core schema that no contract bump catches.
     if businessunitid is not None:
-        name = db.session.execute(
-            db.text('SELECT businessunit FROM businessunits '
-                    'WHERE businessunitid = :bid'),
-            {'bid': businessunitid}).scalar()
-        facts['businessunit'] = name
+        unit = db.session.get(BusinessUnit, businessunitid)
+        facts['businessunit'] = unit.businessunit if unit is not None else None
 
     return {k: str(v).strip().lower() for k, v in facts.items()
             if v is not None and str(v).strip()}
@@ -4099,35 +4097,39 @@ def get_fleet_alerts():
     # computer row means fleet-wide rather than a crash.
     facts = _endpoint_scope_facts(hostname) or {}
 
-    current_app.logger.debug(
-        f'[fleet_alerts] hostname={hostname} asset_id={asset_id} '
-        f'facts={facts} query_results={len(results)}')
-
     alerts = []
     for notification, ntype in results:
-        matches = _alert_targets_endpoint(notification.alertscope, facts)
-        current_app.logger.debug(
-            f'[fleet_alerts] notification_id={notification.notificationid} '
-            f'scope={notification.alertscope} matches={matches}')
-        if not matches:
+        if not _alert_targets_endpoint(notification.alertscope, facts):
             continue
 
         title = notification.notification[:100] if notification.notification else ''
         if '\n' in title:
             title = title.split('\n')[0][:100] or '(no title)'
 
-        alert_data = {
-            'notificationId': notification.notificationid,
+        alerts.append({
+            'notificationid': notification.notificationid,
             'type': ntype.typename,
-            'typeColor': ntype.typecolor or _DEFAULT_ALERT_TYPE_COLOR,
+            'typecolor': ntype.typecolor or _DEFAULT_ALERT_TYPE_COLOR,
             'title': title,
             'message': notification.notification or '',
             'timeout': notification.alerttimeout,
-        }
-        alerts.append(alert_data)
-        current_app.logger.debug(f'[fleet_alerts] added alert: {alert_data}')
+            # TRANSITION, remove once no tray older than 1.7.0 is left in the
+            # fleet: a deployed 1.6.x tray fetches this feed itself and
+            # deserializes case-sensitively, so dropping these two keys makes
+            # every alert vanish on those PCs with no error anywhere. The
+            # lowercase pair above is the contract; these two are a bridge.
+            # Any NEW field is lowercase only.
+            'notificationId': notification.notificationid,
+            'typeColor': ntype.typecolor or _DEFAULT_ALERT_TYPE_COLOR,
+        })
 
-    current_app.logger.debug(f'[fleet_alerts] returning {len(alerts)} alerts')
+    # ONE line, lazily formatted. This runs on every poll from every endpoint,
+    # so an f-string here is formatting work paid at every log level (the cost
+    # HOTFIX 1ed0c4d removed from the collector check-in); per-notification and
+    # per-alert lines multiplied it by the fleet alert count.
+    current_app.logger.debug(
+        '[fleet_alerts] hostname=%s assetid=%s candidates=%d returned=%d',
+        hostname, asset_id, len(results), len(alerts))
     return success_response(alerts)
 
 
@@ -4679,8 +4681,8 @@ def generate_server_certificate():
 
     Expects JSON body:
     {
-        "cn": "localhost",
-        "san": "DNS:localhost,IP:192.168.100.1",
+        "cn": "<hostname or IP agents connect to>",
+        "san": "DNS:<hostname>,IP:<address>",
         "days": 365
     }
 

plugins/endpoints/frontend/views/EnrollmentSettings.vue

@@ -229,6 +229,26 @@
             on record matches. Anything else waits below for an admin.
           </span>
         </label>
+        <label class="number-row">
+          <span>
+            <strong>Pending request limit.</strong> How many requests may wait
+            here at once (setting
+            <code>plugin.endpoints.maxpendingenrollments</code>). At the limit,
+            a PC we have never seen is told to try later, while PCs already in
+            this list keep asking and keep being approved. 0 removes the limit.
+            /enroll takes no login, so this is what stops the queue growing
+            without bound.
+          </span>
+          <input
+            type="number"
+            min="0"
+            step="1"
+            class="input-maxpending"
+            v-model.number="pendingCapInput"
+            :disabled="!isadmin || busy"
+            @change="onPendingCapChange()"
+          />
+        </label>
       </div>
 
       <div class="enroll-tiles">
@@ -645,7 +665,16 @@ function wavePercent(wave) {
 
 // ---- Credentials ------------------------------------------------------
 
-const enrollmentSettings = ref({ sharedkeyallowed: true, autoenroll: true })
+const DEFAULT_PENDING_CAP = 500
+
+const enrollmentSettings = ref({
+  sharedkeyallowed: true,
+  autoenroll: true,
+  maxpendingenrollments: DEFAULT_PENDING_CAP
+})
+// Bound with v-model rather than :value so the field holds what the operator
+// typed until it is accepted or rejected; a one-way binding fights the edit.
+const pendingCapInput = ref(DEFAULT_PENDING_CAP)
 const enrollmentCounts = ref({})
 const credentialRows = ref([])
 const requests = ref([])
@@ -667,8 +696,12 @@ async function loadEnrollment({ quiet = false } = {}) {
     const data = response.data.data || {}
     enrollmentSettings.value = {
       sharedkeyallowed: data.settings?.sharedkeyallowed !== false,
-      autoenroll: data.settings?.autoenroll !== false
+      autoenroll: data.settings?.autoenroll !== false,
+      maxpendingenrollments: Number.isFinite(Number(data.settings?.maxpendingenrollments))
+        ? Number(data.settings.maxpendingenrollments)
+        : DEFAULT_PENDING_CAP
     }
+    pendingCapInput.value = enrollmentSettings.value.maxpendingenrollments
     enrollmentCounts.value = data.counts || {}
     credentialRows.value = data.computers || []
     requests.value = data.requests || []
@@ -738,6 +771,16 @@ function onAutoEnrollToggle(event) {
   })
 }
 
+function onPendingCapChange() {
+  const wanted = Math.trunc(Number(pendingCapInput.value))
+  if (!Number.isFinite(wanted) || wanted < 0) {
+    pendingCapInput.value = enrollmentSettings.value.maxpendingenrollments
+    toast.error('The pending request limit must be 0 or a whole number')
+    return
+  }
+  return saveSettings({ maxpendingenrollments: wanted })
+}
+
 function openApprove(request) {
   const options = []
   if (request.matchedassetid) {
@@ -1126,6 +1169,18 @@ onBeforeUnmount(() => {
   margin: 0.5rem 0;
 }
 
+.number-row {
+  display: flex;
+  align-items: flex-start;
+  gap: 0.5rem;
+  margin: 0.5rem 0;
+}
+
+.number-row input {
+  width: 6rem;
+  flex: none;
+}
+
 .action-cell {
   white-space: nowrap;
   text-align: right;

plugins/endpoints/frontend/views/ServerCertSettings.vue

@@ -164,10 +164,11 @@
             v-model="generateForm.cn"
             type="text"
             class="form-control"
-            placeholder="localhost"
+            placeholder="hostname or IP"
           />
           <p class="field-hint">
-            Hostname or IP that agents use to connect (e.g., shopdb.example.com or 192.168.100.1)
+            Hostname or IP that agents use to connect, exactly as it appears in
+            the agent's base URL.
           </p>
         </div>
 
@@ -178,10 +179,12 @@
             v-model="generateForm.san"
             type="text"
             class="form-control"
-            placeholder="DNS:localhost,IP:192.168.100.1"
+            placeholder="DNS:hostname,IP:address"
           />
           <p class="field-hint">
-            Comma-separated list: DNS:hostname or IP:address (e.g., DNS:localhost,IP:192.168.100.1,DNS:shopdb.local)
+            Comma-separated list of DNS:hostname and IP:address entries. Include
+            every name and address a bay may use to reach this server; a name
+            that is missing here fails certificate validation on the endpoint.
           </p>
         </div>
 
@@ -355,9 +358,13 @@ const liveLoading = ref(false)
 const thumbprintCopied = ref(false)
 const pinCopied = ref(false)
 
+// No baked host or address (ADR-015: no site literals in product code). One
+// site's LAN address was shipped here as the default SAN, which every other
+// site would have generated a certificate for. The operator types what this
+// server is actually reached by; the form refuses to submit empty.
 const generateForm = ref({
-  cn: 'localhost',
-  san: 'DNS:localhost,IP:192.168.100.1',
+  cn: '',
+  san: '',
   days: 365
 })
 

plugins/endpoints/frontend/views/enrollmentSettings.spec.js

@@ -54,7 +54,8 @@ function enrollmentPayload() {
   return {
     data: {
       data: {
-        settings: { sharedkeyallowed: true, autoenroll: true },
+        settings: { sharedkeyallowed: true, autoenroll: true,
+                    maxpendingenrollments: 500 },
         counts: { enrolled: 40, notenrolled: 58, pendingcomputers: 1, pending: 2 },
         computers: [
           { assetid: 21, hostname: 'PC-ENROLLED', serialnumber: 'S1', state: 'enrolled',
@@ -325,4 +326,20 @@ describe('EnrollmentSettings', () => {
     expect(api.put).not.toHaveBeenCalled()
     expect(toggle.element.checked).toBe(true)
   })
+
+  it('saves a new pending request limit', async () => {
+    const wrapper = await render()
+    await openCredentials(wrapper)
+    const field = wrapper.find('.input-maxpending')
+    expect(field.element.value).toBe('500')
+    await field.setValue(25)
+    await field.trigger('change')
+    await flushPromises()
+    expect(api.put).toHaveBeenCalledWith('/endpoints/enrollment/settings',
+      { maxpendingenrollments: 25 })
+  })
+
+  // The negative case is proved server side (tests/test_plugins/
+  // test_endpoints_enroll_limits.py: PUT with -1 is a 400). Asserting it here
+  // would only be asserting what vue-test-utils does to a number input.
 })

plugins/endpoints/manifest.json

@@ -4,7 +4,7 @@
     "description": "Endpoint RMM telemetry: metrics, services, inventory, and curated Windows event records reported by the ShopDB agent",
     "author": "ShopDB Team",
     "dependencies": ["computers"],
-    "core_version": ">=0.20.0,<1.0.0",
+    "core_version": ">=0.34.0,<1.0.0",
     "api_prefix": "/api/endpoints",
     "provides": {
         "features": [

plugins/endpoints/plugin.py

@@ -21,6 +21,7 @@ from pathlib import Path
 from typing import Dict, List, Optional, Type
 
 from flask import Flask, Blueprint, g
+from sqlalchemy.exc import IntegrityError
 
 from shopdb.plugins.base import BasePlugin, PluginMeta
 from shopdb.api import db
@@ -325,6 +326,19 @@ class EndpointsPlugin(BasePlugin):
                 'description': 'True once an operator has minted a UI-managed '
                                'endpoint agent key from the settings page.',
             },
+            {
+                'key': 'plugin.endpoints.maxpendingenrollments',
+                'value': '500',
+                'valuetype': 'integer',
+                'category': 'plugin.endpoints',
+                'description': 'How many enrollment requests may wait for an '
+                               'admin at once. At the cap, a request from a '
+                               'hostname and serial never seen before is '
+                               'refused; PCs already in the queue keep asking '
+                               'and keep being approved. 0 removes the cap. '
+                               'POST /enroll takes no auth, so this is what '
+                               'stops the pending queue growing without limit.',
+            },
         ]
 
     def get_settings_cards(self) -> List[Dict]:
@@ -1285,23 +1299,110 @@ class EndpointsPlugin(BasePlugin):
                     f'delete count mismatch: expected {len(to_remove)}, '
                     f'got {deleted_count} (concurrent update?)')
 
-        # Add new apps
-        for fp in to_add:
+        # Add new apps.
+        #
+        # The unique index on (assetid, appfingerprint) DETECTS a concurrent
+        # insert of the same row; it does not RESOLVE one. Two check-ins for the
+        # same PC can overlap (an agent retry over a slow request, two cloned
+        # machines sharing a hostname, the backfill script against a live
+        # fleet), and without a savepoint the IntegrityError from the flush
+        # below sinks the whole check-in - state, events and hardware - which is
+        # exactly the loss the delete-mismatch branch above was written to
+        # avoid. So: flush inside a savepoint, and on a collision roll back only
+        # the savepoint, re-read what is there now, and insert what is still
+        # missing. Same shape as the health block in apply_collector_payload.
+        if to_add or to_remove:
+            # SAFETY: Flush to detect constraint violations before continuing.
+            # Only flush if we made changes (99% of check-ins change nothing).
+            try:
+                with db.session.begin_nested():
+                    self._insert_applications(assetid, incoming, to_add)
+                    db.session.flush()
+            except IntegrityError:
+                # The context manager has already released the savepoint, so
+                # the deletes and everything before them survive.
+                current_app.logger.warning(
+                    '[collector] asset_id=%s application insert collided with a '
+                    'concurrent check-in (%d new fingerprint(s)); reconciling',
+                    assetid, len(to_add))
+                return self._reconcile_applications(assetid, incoming, to_add,
+                                                    warnings)
+
+        return len(to_add)
+
+    @staticmethod
+    def _insert_applications(assetid: int, incoming: Dict, fingerprints) -> list:
+        """Add one EndpointApplication per fingerprint. Returns the new rows."""
+        rows = []
+        for fp in fingerprints:
             app_data = incoming[fp]
-            db.session.add(EndpointApplication(
+            row = EndpointApplication(
                 assetid=assetid,
                 displayname=app_data['displayname'],
                 displayversion=app_data['displayversion'],
                 publisher=app_data['publisher'],
                 appfingerprint=fp,
-            ))
+            )
+            rows.append(row)
+            db.session.add(row)
+        return rows
 
-        # SAFETY: Flush to detect constraint violations before continuing
-        # Only flush if we made changes (skip when unchanged apps = 99% of cases)
-        if to_add or to_remove:
+    def _reconcile_applications(self, assetid: int, incoming: Dict, to_add,
+                                warnings: List[str]) -> int:
+        """Re-insert after a unique-index collision, skipping what now exists.
+
+        Called only when the bulk insert lost a race. Re-reads the asset's
+        fingerprints, drops the ones another writer has since committed, and
+        inserts the rest ONE ROW PER SAVEPOINT so a second collision skips that
+        row instead of failing the batch again. Per-row savepoints are only
+        paid on this path; the common case never reaches it.
+
+        Never raises: a check-in must commit its telemetry even when the
+        application inventory cannot be reconciled. Whatever is missed is
+        picked up by the next check-in, which reports the full list again.
+        """
+        from flask import current_app
+
+        # The failed flush rolled the savepoint back, which restores the added
+        # rows to pending. Expunge them, or the autoflush on the re-read below
+        # retries the insert that just collided - this time with no savepoint
+        # under it.
+        for row in list(db.session.new):
+            if isinstance(row, EndpointApplication) and row.assetid == assetid:
+                db.session.expunge(row)
+
+        existing = {app.appfingerprint
+                    for app in (EndpointApplication.query
+                                .filter_by(assetid=assetid)
+                                .all())
+                    if app.appfingerprint}
+
+        stored = 0
+        skipped = 0
+        for fp in to_add:
+            if fp in existing:
+                skipped += 1
+                continue
+            try:
+                with db.session.begin_nested():
+                    self._insert_applications(assetid, incoming, [fp])
                     db.session.flush()
+                stored += 1
+            except IntegrityError:
+                for row in list(db.session.new):
+                    if (isinstance(row, EndpointApplication)
+                            and row.appfingerprint == fp):
+                        db.session.expunge(row)
+                skipped += 1
 
-        return len(to_add)
+        current_app.logger.warning(
+            '[collector] asset_id=%s application_insert_conflict: stored=%d '
+            'skipped=%d of %d new (concurrent check-in, telemetry kept)',
+            assetid, stored, skipped, len(to_add))
+        warnings.append(
+            'application insert conflict: stored {}, skipped {} of {} new '
+            '(concurrent update?)'.format(stored, skipped, len(to_add)))
+        return stored
 
     def apply_collector_payload(self, payload: Dict) -> Dict:
         """Idempotently upsert one endpoint's state and append its events.

shopdb/__init__.py

@@ -130,7 +130,16 @@ from .plugins import plugin_manager
 # for an enrolled PC (401 SHARED_KEY_REFUSED), and maps a PermissionError from
 # apply_collector_payload to 403 ASSET_MISMATCH. All optional, so a plugin that
 # declares none of it is unaffected. Additive, minor bump.
-__contract_version__ = '0.33.0'
+# 0.34.0 adds BusinessUnit to the core model surface. It was the one thing an
+# asset carries that a plugin could not name: the endpoints fleet-alert scope
+# resolves a business unit per poll, and with no contract name for it the route
+# had dropped to db.text('SELECT businessunit FROM businessunits ...') against a
+# core table. That workaround respects the letter of ADR-002 and defeats its
+# point - a raw query is hidden coupling to the core schema that no contract
+# bump can catch, and it is the kind of thing that gets copied. Exposing the
+# model closes the gap instead of leaving the workaround as the worked example.
+# Additive name on the surface, minor bump.
+__contract_version__ = '0.34.0'
 
 # Product release version (see ADR-007). The product version and the
 # plugin-contract version above are distinct series with independent

shopdb/api/__init__.py

@@ -32,6 +32,7 @@ from shopdb.core.models import (
     Asset,
     AssetType,
     AssetStatus,
+    BusinessUnit,
     Vendor,
     Model,
     Communication,
@@ -417,6 +418,7 @@ __all__ = [
     'Asset',
     'AssetType',
     'AssetStatus',
+    'BusinessUnit',
     'Vendor',
     'Model',
     'Communication',

shopdb/config.py

@@ -10,6 +10,18 @@ class ConfigError(Exception):
     """Raised when required configuration is missing or unsafe."""
 
 
+# Connection pool per worker process. Env-driven so a site can size it against
+# its own MySQL max_connections without editing code; see the arithmetic in
+# docker-compose.yml next to the worker count.
+#
+# Deliberately NOT named SQLALCHEMY_POOL_SIZE and friends: those were
+# Flask-SQLAlchemy 2.x config keys, removed in 3.x, and reusing the names would
+# suggest the extension still reads them. It reads SQLALCHEMY_ENGINE_OPTIONS.
+DB_POOL_SIZE = int(os.environ.get('DB_POOL_SIZE', 10))
+DB_MAX_OVERFLOW = int(os.environ.get('DB_MAX_OVERFLOW', 10))
+DB_POOL_TIMEOUT = int(os.environ.get('DB_POOL_TIMEOUT', 5))
+
+
 def _required_env(varname):
     """Read an env var; raise ConfigError if missing or empty."""
     value = os.environ.get(varname)
@@ -32,9 +44,32 @@ class Config:
         'mysql+pymysql://root:password@localhost:3306/shopdb_flask',
     )
     SQLALCHEMY_TRACK_MODIFICATIONS = False
+
+    # Connection pool, sized on purpose rather than left to SQLAlchemy.
+    #
+    # The defaults (pool_size 5, max_overflow 10, pool_timeout 30) were never
+    # chosen: they simply applied, and nothing in the repo related them to the
+    # gunicorn worker count or to the MySQL max_connections ceiling. The
+    # arithmetic that ties the three together is written next to the worker
+    # count in docker-compose.yml.
+    #
+    # Under gevent a pooled connection is held for the duration of whatever the
+    # greenlet is doing, so a request that sits inside a slow external HTTP call
+    # WITH THE SESSION OPEN pins a connection for that whole call. Hundreds of
+    # greenlets can enter such a path at once and drain the pool, at which point
+    # every other request blocks on checkout. Hence the fail-fast timeout: 5
+    # seconds, not 30. A pool that is empty for five seconds is not going to be
+    # fine in thirty, and a fast 500 is diagnosable where a 30-second hang looks
+    # like a dead database.
+    #
+    # See PLUGIN-DEVELOPMENT / the plugin docs: a plugin must not hold a DB
+    # session across external I/O.
     SQLALCHEMY_ENGINE_OPTIONS = {
         'pool_pre_ping': True,
         'pool_recycle': 300,
+        'pool_size': DB_POOL_SIZE,
+        'max_overflow': DB_MAX_OVERFLOW,
+        'pool_timeout': DB_POOL_TIMEOUT,
         # Pin the client connection to utf8mb4 so reads/writes match the
         # utf8mb4 schema regardless of the server default charset. TestingConfig
         # (SQLite) replaces this whole dict, so the charset arg never reaches it.
@@ -192,6 +227,13 @@ class TestingConfig(Config):
     # Off by default so login-heavy fixtures do not trip the limiter; tests
     # that exercise rate limiting flip it on via app.config override.
     AUTH_RATELIMIT_ENABLED = False
+    # Same reason: the enrollment suites post to /enroll dozens of times, and
+    # the limiter counts in a process-wide cache that outlives a test, so with
+    # it on by default a budget spent in one module surfaces as an unrelated
+    # failure in another - and which modules share a worker depends on the
+    # xdist distribution, so the failure moves around. The tests that are about
+    # the limiter turn it on themselves.
+    ENDPOINTS_ENROLL_RATELIMIT_ENABLED = False
     SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
     SQLALCHEMY_ENGINE_OPTIONS = {
         'connect_args': {'check_same_thread': False},

shopdb/core/api/plugins.py

@@ -43,13 +43,23 @@ def list_enabled_plugins():
 
 
 @plugins_bp.route('/settings-routes', methods=['GET'])
-@jwt_required(optional=True)
+@require_permission('settings.view')
 def get_plugin_settings_routes():
     """Return settings routes contributed by all enabled plugins.
 
     Plugins can register routes to add tabs/pages to settings hubs like
     Model Catalog, Asset Config, etc. Frontend router merges these dynamically.
 
+    Gated on settings.view, the same permission list_plugins carries. It was
+    optional-auth and listed as a public read, on the stated reasoning that
+    unauthenticated pages (kiosks, TV displays) register plugin tabs before
+    login. Nothing does: the only caller in the frontend is
+    views/settings/ModelCatalogHub.vue, which sits behind requiresAuth and
+    requiresAdmin, and no kiosk route touches it. Open, it handed an anonymous
+    caller the enabled plugin list, the settings hub layout and the internal Vue
+    component paths - free reconnaissance about which modules a site runs. The
+    login page never calls this, so gating it costs nothing pre-login.
+
     Returns: { routes: [{hub, path, name, tab, component}, ...] }
     """
     pm = current_app.extensions.get('plugin_manager')

shopdb/plugins/base.py

@@ -1,4 +1,17 @@
-"""Base plugin class that all plugins must inherit from."""
+"""Base plugin class that all plugins must inherit from.
+
+One rule that is not expressible as a method signature, so it lives here where
+a plugin author reads it: A PLUGIN MUST NOT HOLD A DB SESSION ACROSS EXTERNAL
+I/O. Finish with the database, then make the HTTP call / run the subprocess /
+touch the share, then open a fresh session to write the result. Under gevent a
+slow external call does not block its worker, but it does pin one of that
+worker's pooled database connections for its whole duration, and enough
+concurrent greenlets on such a path drain the pool and stall every other
+request in the process - the whole site, every plugin, with the CPU idle and
+MySQL healthy. See CONTRIBUTING.md ("A plugin must not hold a DB session across
+external I/O") for the shape to write, and docker-compose.yml for how the pool
+relates to the worker count and to MySQL max_connections.
+"""
 
 from abc import ABC, abstractmethod
 from typing import List, Dict, Optional, Type

tests/test_core/test_asset_type_descriptor_enforcement.py

@@ -0,0 +1,79 @@
+"""A descriptor's edit_permission must be real, not advisory.
+
+`get_asset_type_descriptor` declares `edit_permission`, and
+/api/pluginui/asset-type-descriptors hands it to the Asset Type Hub, which
+renders generic CRUD against the declared `subtype_endpoint`. Nothing in the
+contract makes the plugin actually enforce that permission on its own routes.
+
+Every shipped plugin does today. The risk is structural: a plugin author can
+declare a permission, forget the decorator, and ship a UI that looks locked over
+an API that is open - and the contract invites exactly that mistake. So walk the
+descriptors the API really returns and prove each named endpoint refuses a
+caller who holds nothing.
+
+A signed-in caller with no permissions, not an anonymous one: anonymous would
+pass on the 401 and prove nothing about authorization.
+"""
+
+import pytest
+
+
+DESCRIPTORS_URL = '/api/pluginui/asset-type-descriptors'
+
+# The write methods the Asset Type Hub drives through subtype_endpoint. GET is
+# not here on purpose: a list may legitimately need only a read permission,
+# while edit_permission is about changing a subtype.
+WRITE_CALLS = (
+    ('post', '{endpoint}'),
+    ('put', '{endpoint}/1'),
+    ('delete', '{endpoint}/1'),
+)
+
+
+def _descriptors(client, auth_headers):
+    response = client.get(DESCRIPTORS_URL, headers=auth_headers)
+    assert response.status_code == 200, response.get_data(as_text=True)
+    return response.get_json()['data']
+
+
+def test_there_are_descriptors_to_check(client, db, auth_headers):
+    """A silently empty list would make the sweep below vacuous."""
+    assert _descriptors(client, auth_headers), (
+        'no plugin returned an asset type descriptor; the enforcement sweep '
+        'below would pass without checking anything')
+
+
+def test_every_descriptor_endpoint_refuses_an_unpermissioned_caller(
+        client, db, auth_headers, member_headers):
+    """The permission a descriptor advertises has to be the one the API asks
+    for. A route that answers anything but 403 here is a lock drawn on a door
+    that is not locked."""
+    open_routes = []
+    for descriptor in _descriptors(client, auth_headers):
+        endpoint = '/api' + descriptor['subtype_endpoint']
+        for method, template in WRITE_CALLS:
+            path = template.format(endpoint=endpoint)
+            call = getattr(client, method)
+            response = call(path, json={'name': 'x'}, headers=member_headers)
+            if response.status_code != 403:
+                open_routes.append(
+                    '{0} declares {1} but {2} {3} answered {4}'.format(
+                        descriptor.get('plugin'),
+                        descriptor.get('edit_permission'),
+                        method.upper(), path, response.status_code))
+
+    assert not open_routes, (
+        'These descriptors advertise protection the API does not enforce. '
+        'Decorate the subtype routes with @require_permission for the '
+        'permission the descriptor names:\n  ' + '\n  '.join(open_routes))
+
+
+@pytest.mark.parametrize('field', ['assettype', 'subtype_endpoint',
+                                   'edit_permission'])
+def test_every_descriptor_names_what_the_sweep_needs(client, db, auth_headers,
+                                                     field):
+    """The sweep above can only check a descriptor that says what to check."""
+    for descriptor in _descriptors(client, auth_headers):
+        assert descriptor.get(field), (
+            '{0} returned a descriptor with no {1}'.format(
+                descriptor.get('plugin'), field))

tests/test_core/test_plugin_settings_routes.py

@@ -1,7 +1,8 @@
-"""GET /api/plugins/settings-routes survives a plugin whose hook raises."""
+"""GET /api/plugins/settings-routes: isolation, and who may read it."""
 
 
-def test_a_raising_plugin_is_skipped_not_fatal(app, client, db, monkeypatch):
+def test_a_raising_plugin_is_skipped_not_fatal(app, client, db, monkeypatch,
+                                              auth_headers):
     manager = app.extensions.get('plugin_manager')
     plugins = manager.get_all_plugins()
     enabled = [name for name in plugins if manager.registry.is_enabled(name)]
@@ -21,8 +22,26 @@ def test_a_raising_plugin_is_skipped_not_fatal(app, client, db, monkeypatch):
     monkeypatch.setattr(plugins[healthy], 'get_settings_routes', contribute,
                         raising=False)
 
-    response = client.get('/api/plugins/settings-routes')
+    response = client.get('/api/plugins/settings-routes',
+                          headers=auth_headers)
 
     assert response.status_code == 200, response.get_json()
     names = [route['name'] for route in response.get_json()['data']['routes']]
     assert 'probe-route' in names
+
+
+def test_an_anonymous_caller_is_refused(client, db):
+    """It names the enabled plugins, the settings hub layout and the internal
+    component paths. That is reconnaissance, and nothing needs it before login -
+    its only caller is the Model Catalog hub, behind auth and admin."""
+    response = client.get('/api/plugins/settings-routes')
+
+    assert response.status_code in (401, 403, 422), response.get_data(as_text=True)
+
+
+def test_a_caller_without_settings_view_is_refused(client, db, member_headers):
+    """Signed in is not enough: the gate is the permission, not the session."""
+    response = client.get('/api/plugins/settings-routes',
+                          headers=member_headers)
+
+    assert response.status_code == 403, response.get_data(as_text=True)

tests/test_core/test_public_read_surface.py

@@ -26,11 +26,12 @@ PUBLIC_ENDPOINTS = {
     'setup.needs_admin',
     'auth.get_current_user',
     'plugins.list_enabled_plugins',
-    'plugins.get_plugin_settings_routes',  # plugin-contributed settings routes
-                                           # for dynamic UI extension; must answer
-                                           # unauthenticated pages (kiosks, TV
-                                           # displays) so they can register plugin
-                                           # tabs before login
+    # plugins.get_plugin_settings_routes was here, on the reasoning that
+    # unauthenticated pages had to register plugin tabs before login. Nothing
+    # did: its only caller is the Model Catalog settings hub, which is behind
+    # requiresAuth + requiresAdmin. Open, it named the enabled plugins, the
+    # settings hub layout and the internal component paths to anybody who
+    # asked, so it is now gated on settings.view like the plugin list beside it.
     'dashboard.get_navigation',      # filtered per caller; a guest sees only
                                      # what the guest role can reach
     'dashboard.get_capabilities',    # booleans about the CALLER, used to decide

tests/test_plugins/test_endpoints_app_conflict.py

@@ -0,0 +1,165 @@
+"""A concurrent application insert must not sink the whole check-in.
+
+The unique index on (assetid, appfingerprint) detects a second writer inserting
+the same application row; it does not resolve one. Before the savepoint went in,
+the IntegrityError from the insert flush escaped generic_collect, reached the
+global handler, rolled back the WHOLE check-in - state, events and hardware -
+and answered the agent with operator-facing 409 form copy.
+
+The insert now behaves like the delete-count mismatch beside it: warn, keep the
+telemetry, and let the next check-in (which reports the full list again) settle
+the inventory.
+"""
+
+import hashlib
+
+from sqlalchemy.exc import IntegrityError
+
+from shopdb.core.models import Asset, AssetType
+from plugins.endpoints.models import (
+    EndpointApplication, EndpointEvent, EndpointState,
+)
+from plugins.endpoints.plugin import EndpointsPlugin
+
+
+def _fingerprint(name, version, publisher):
+    data = '{}|{}|{}'.format(name or '', version or '', publisher or '')
+    return hashlib.sha256(data.encode('utf-8')).hexdigest()
+
+
+def _computer(db, hostname):
+    from plugins.computers.models import Computer
+
+    assettype = AssetType.query.filter_by(assettype='computer').first()
+    if assettype is None:
+        assettype = AssetType(assettype='computer')
+        db.session.add(assettype)
+        db.session.flush()
+    asset = Asset(assetnumber=hostname, name=hostname,
+                  assettypeid=assettype.assettypeid, isactive=True)
+    db.session.add(asset)
+    db.session.flush()
+    db.session.add(Computer(assetid=asset.assetid, hostname=hostname))
+    db.session.commit()
+    return asset.assetid
+
+
+def _payload(hostname, applications):
+    return {
+        'hostname': hostname,
+        'pendingreboot': True,
+        'applications': applications,
+        'events': [{
+            'eventid': 4625,
+            'eventrecordid': 501,
+            'eventchannel': 'Security',
+            'eventsource': 'Microsoft-Windows-Security-Auditing',
+            'eventmessage': 'Failed logon',
+        }],
+    }
+
+
+def test_insert_conflict_keeps_the_checkin(db, monkeypatch):
+    """A REAL IntegrityError on the insert flush leaves the check-in intact.
+
+    The first (bulk) insert adds a genuine duplicate row, so the flush raises on
+    the unique index exactly as a concurrent check-in would make it. The
+    reconcile then re-reads and re-inserts, and state plus events still commit.
+    """
+    assetid = _computer(db, 'APPRACE01')
+    plugin = EndpointsPlugin()
+
+    real_insert = EndpointsPlugin._insert_applications
+    calls = []
+
+    def conflicting_insert(assetid_arg, incoming, fingerprints):
+        rows = real_insert(assetid_arg, incoming, fingerprints)
+        calls.append(list(fingerprints))
+        if len(calls) == 1:
+            # The conflicting row a second writer would have committed.
+            first = rows[0]
+            db.session.add(EndpointApplication(
+                assetid=first.assetid,
+                displayname=first.displayname,
+                displayversion=first.displayversion,
+                publisher=first.publisher,
+                appfingerprint=first.appfingerprint,
+            ))
+        return rows
+
+    monkeypatch.setattr(EndpointsPlugin, '_insert_applications',
+                        staticmethod(conflicting_insert))
+
+    applications = [
+        {'displayname': 'Alpha', 'displayversion': '1.0', 'publisher': 'ACME',
+         'fingerprint': _fingerprint('Alpha', '1.0', 'ACME')},
+        {'displayname': 'Beta', 'displayversion': '2.0', 'publisher': 'ACME',
+         'fingerprint': _fingerprint('Beta', '2.0', 'ACME')},
+    ]
+
+    result = plugin.apply_collector_payload(_payload('APPRACE01', applications))
+    db.session.commit()
+
+    # The collision was detected and reported rather than raised.
+    assert len(calls) > 1, 'the bulk insert should have collided and reconciled'
+    assert any('application insert conflict' in w for w in result['warnings'])
+
+    # The telemetry the check-in carried is still there.
+    state = EndpointState.query.filter_by(assetid=assetid).first()
+    assert state is not None
+    assert state.pendingreboot is True
+    assert state.lastreportedat is not None
+    assert EndpointEvent.query.filter_by(assetid=assetid).count() == 1
+
+    # And the inventory reconciled rather than being abandoned.
+    stored = {row.appfingerprint for row
+              in EndpointApplication.query.filter_by(assetid=assetid).all()}
+    assert stored == {app['fingerprint'] for app in applications}
+
+
+def test_reconcile_skips_fingerprints_that_now_exist(db):
+    """Reconcile inserts only what is missing and counts the rest as skipped."""
+    assetid = _computer(db, 'APPRACE02')
+    plugin = EndpointsPlugin()
+
+    taken = _fingerprint('Alpha', '1.0', 'ACME')
+    missing = _fingerprint('Beta', '2.0', 'ACME')
+    db.session.add(EndpointApplication(
+        assetid=assetid, displayname='Alpha', displayversion='1.0',
+        publisher='ACME', appfingerprint=taken))
+    db.session.commit()
+
+    incoming = {
+        taken: {'displayname': 'Alpha', 'displayversion': '1.0',
+                'publisher': 'ACME'},
+        missing: {'displayname': 'Beta', 'displayversion': '2.0',
+                  'publisher': 'ACME'},
+    }
+    warnings = []
+    stored = plugin._reconcile_applications(assetid, incoming,
+                                            [taken, missing], warnings)
+    db.session.commit()
+
+    assert stored == 1
+    assert warnings == ['application insert conflict: stored 1, skipped 1 '
+                        'of 2 new (concurrent update?)']
+    rows = EndpointApplication.query.filter_by(assetid=assetid).all()
+    assert len(rows) == 2
+    assert {r.appfingerprint for r in rows} == {taken, missing}
+
+
+def test_the_unique_index_is_really_there(db):
+    """The whole fix rests on the constraint raising. Prove it does."""
+    assetid = _computer(db, 'APPRACE03')
+    fingerprint = _fingerprint('Alpha', '1.0', 'ACME')
+    for _ in range(2):
+        db.session.add(EndpointApplication(
+            assetid=assetid, displayname='Alpha', displayversion='1.0',
+            publisher='ACME', appfingerprint=fingerprint))
+    raised = False
+    try:
+        db.session.flush()
+    except IntegrityError:
+        raised = True
+        db.session.rollback()
+    assert raised, 'endpointapplications has no unique (assetid, appfingerprint)'

tests/test_plugins/test_endpoints_credentials.py

@@ -688,7 +688,8 @@ def test_enrollment_listing_and_settings(client, db, app, keyheaders,
     assert response.status_code == 200, response.get_data(as_text=True)
     body = response.get_json()
     data = body['data']
-    assert data['settings'] == {'sharedkeyallowed': True, 'autoenroll': True}
+    assert data['settings'] == {'sharedkeyallowed': True, 'autoenroll': True,
+                                'maxpendingenrollments': 500}
     assert data['counts'] == {'enrolled': 1, 'notenrolled': 1,
                               'pendingcomputers': 1, 'pending': 1}
     states = {row['hostname']: row['state'] for row in data['computers']}
@@ -709,7 +710,8 @@ def test_enrollment_listing_and_settings(client, db, app, keyheaders,
                      json={'sharedkeyallowed': False}, headers=auth_headers)
     assert put.status_code == 200
     assert put.get_json()['data'] == {'sharedkeyallowed': False,
-                                      'autoenroll': True}
+                                      'autoenroll': True,
+                                      'maxpendingenrollments': 500}
     bad = client.put('/api/endpoints/enrollment/settings',
                      json={'autoenroll': 'yes'}, headers=auth_headers)
     assert bad.status_code == 400

tests/test_plugins/test_endpoints_enroll_limits.py

@@ -0,0 +1,220 @@
+"""The two bounds on POST /api/endpoints/enroll.
+
+The route takes no auth on purpose - a PC installed without the shared key must
+still be able to reach an admin - and it writes a row for every new
+(hostname, serialnumber) pair. That is an unauthenticated caller writing to the
+database in a loop, and it is also a denial of the enrollment workflow: junk
+pending rows bury the real requests an admin has to approve.
+
+Two bounds, and they answer different questions. The per-IP fixed-window limiter
+bounds how FAST; the settings-backed cap bounds how MANY rows exist. Neither may
+touch a PC already in the workflow, so the cap refuses only a hostname and
+serial never seen before and an existing pending or approved row still answers
+exactly as it did.
+"""
+
+import pytest
+
+from shopdb.core.models import Asset, AssetType, Setting
+
+from plugins.endpoints.api.enrollment import PENDING_CAP_SETTING
+from plugins.endpoints.models import EndpointEnrollRequest
+
+
+ENROLL_URL = '/api/endpoints/enroll'
+NONCE = 'n' * 40
+
+
+@pytest.fixture
+def limits(app):
+    """Both bounds on, at their shipped defaults, restored afterwards.
+
+    The limiter counts in the cache extension, which is process-wide and
+    outlives a test. Any other test in the same worker that posts to /enroll
+    spends budget in the same window and IP bucket, so a count-exactly test has
+    to start from a clean counter - and leave one behind, since it deliberately
+    fills a bucket. Which tests share a worker depends on the xdist
+    distribution, so without this the failure is intermittent and looks
+    unrelated to whatever changed.
+    """
+    from shopdb.api import cache
+
+    previous = {key: app.config.get(key) for key in (
+        'ENDPOINTS_ENROLL_RATELIMIT_ENABLED',
+        'ENDPOINTS_ENROLL_RATELIMIT_MAX',
+        'ENDPOINTS_ENROLL_RATELIMIT_WINDOW_SECONDS')}
+    app.config['ENDPOINTS_ENROLL_RATELIMIT_ENABLED'] = True
+    with app.app_context():
+        cache.clear()
+    yield app
+    with app.app_context():
+        cache.clear()
+    for key, value in previous.items():
+        if value is None:
+            app.config.pop(key, None)
+        else:
+            app.config[key] = value
+
+
+def _computer(db, hostname, serialnumber):
+    from plugins.computers.models import Computer
+
+    atype = AssetType.query.filter_by(assettype='computer').first()
+    if atype is None:
+        atype = AssetType(assettype='computer')
+        db.session.add(atype)
+        db.session.flush()
+    asset = Asset(assetnumber=hostname, assettypeid=atype.assettypeid,
+                  serialnumber=serialnumber, isactive=True)
+    db.session.add(asset)
+    db.session.flush()
+    db.session.add(Computer(assetid=asset.assetid, hostname=hostname))
+    db.session.commit()
+    return asset.assetid
+
+
+def _body(hostname, serialnumber=None, nonce=NONCE):
+    return {'hostname': hostname,
+            'serialnumber': serialnumber or (hostname + '-SER'),
+            'enrollnonce': nonce, 'agentversion': '1.20.0'}
+
+
+def _cap(db, value):
+    Setting.set(PENDING_CAP_SETTING, value, valuetype='integer',
+                category='plugin.endpoints')
+    db.session.commit()
+
+
+def _pending(db, hostname, serialnumber=None, nonce=NONCE,
+             status=EndpointEnrollRequest.STATUS_PENDING):
+    from datetime import datetime, timezone
+
+    now = datetime.now(timezone.utc).replace(tzinfo=None)
+    row = EndpointEnrollRequest(
+        hostname=hostname, serialnumber=serialnumber or (hostname + '-SER'),
+        enrollnonce=nonce, status=status, requestedat=now)
+    db.session.add(row)
+    db.session.commit()
+    return row
+
+
+def _code(response):
+    return response.get_json()['data']['error']['code']
+
+
+# -- rate limit --------------------------------------------------------------
+
+def test_the_limit_triggers_and_answers_the_standard_shape(client, db, limits):
+    """Over budget is a 429 RATE_LIMITED, the same shape as every other
+    limiter in the product."""
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_MAX'] = 3
+    # An hour, so the fixed window cannot roll over mid-test and reset the
+    # count. The test is about the budget, not about the clock.
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_WINDOW_SECONDS'] = 3600
+
+    codes = []
+    for index in range(5):
+        response = client.post(ENROLL_URL, json=_body('BAY-%02d' % index))
+        codes.append(response.status_code)
+
+    assert codes[:3] == [202, 202, 202], codes
+    assert codes[3:] == [429, 429], codes
+
+    refused = client.post(ENROLL_URL, json=_body('BAY-99'))
+    assert refused.status_code == 429
+    assert _code(refused) == 'RATE_LIMITED'
+
+
+def test_the_limit_is_off_when_the_config_says_so(client, db, limits):
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_ENABLED'] = False
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_MAX'] = 1
+
+    for index in range(4):
+        response = client.post(ENROLL_URL, json=_body('OFF-%02d' % index))
+        assert response.status_code == 202, response.get_data(as_text=True)
+
+
+# -- pending cap -------------------------------------------------------------
+
+def test_the_cap_refuses_a_new_row(client, db, limits):
+    """At the cap, a hostname and serial nobody has seen is told to try later,
+    and no row is written for it."""
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_ENABLED'] = False
+    _cap(db, 2)
+    _pending(db, 'OLD-01')
+    _pending(db, 'OLD-02')
+
+    response = client.post(ENROLL_URL, json=_body('NEW-01'))
+
+    assert response.status_code == 503, response.get_data(as_text=True)
+    assert _code(response) == 'ENROLL_PENDING_FULL'
+    assert EndpointEnrollRequest.query.filter_by(hostname='NEW-01').count() == 0
+    assert EndpointEnrollRequest.query.count() == 2
+
+
+def test_the_cap_still_serves_an_existing_pending_row(client, db, limits):
+    """A PC already in the queue keeps asking and keeps being answered - the cap
+    must not lock the fleet out of the workflow it is protecting."""
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_ENABLED'] = False
+    _cap(db, 1)
+    row = _pending(db, 'OLD-01')
+
+    response = client.post(ENROLL_URL, json=_body('OLD-01'))
+
+    assert response.status_code == 202, response.get_data(as_text=True)
+    assert response.get_json()['data']['status'] == 'pending'
+    db.session.refresh(row)
+    assert row.lastseenat is not None
+
+
+def test_the_cap_still_issues_to_an_approved_row(client, db, limits):
+    """An approved request hands out its credential at the cap, because that
+    row is the admin's decision and refusing it would strand the PC."""
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_ENABLED'] = False
+    assetid = _computer(db, 'BAY-07', 'BAY-07-SER')
+    _cap(db, 1)
+    _pending(db, 'FILLER-01')
+    approved = _pending(db, 'BAY-07',
+                        status=EndpointEnrollRequest.STATUS_APPROVED)
+    approved.matchedassetid = assetid
+    db.session.commit()
+
+    response = client.post(ENROLL_URL, json=_body('BAY-07'))
+
+    assert response.status_code == 201, response.get_data(as_text=True)
+    assert response.get_json()['data']['credential'].startswith('sdbe_')
+
+
+def test_the_cap_setting_is_honoured(client, db, limits):
+    """Raise the setting and the same request that was refused goes through;
+    0 removes the cap entirely."""
+    limits.config['ENDPOINTS_ENROLL_RATELIMIT_ENABLED'] = False
+    _cap(db, 1)
+    _pending(db, 'OLD-01')
+
+    refused = client.post(ENROLL_URL, json=_body('NEW-01'))
+    assert refused.status_code == 503
+
+    _cap(db, 5)
+    allowed = client.post(ENROLL_URL, json=_body('NEW-01'))
+    assert allowed.status_code == 202, allowed.get_data(as_text=True)
+
+    _cap(db, 0)
+    uncapped = client.post(ENROLL_URL, json=_body('NEW-02'))
+    assert uncapped.status_code == 202, uncapped.get_data(as_text=True)
+
+
+def test_the_cap_is_editable_through_the_enrollment_settings_route(
+        client, db, auth_headers):
+    """A setting nobody can change is a constant with extra steps."""
+    response = client.put('/api/endpoints/enrollment/settings',
+                          json={'maxpendingenrollments': 42},
+                          headers=auth_headers)
+
+    assert response.status_code == 200, response.get_data(as_text=True)
+    assert response.get_json()['data']['maxpendingenrollments'] == 42
+    assert int(Setting.get(PENDING_CAP_SETTING)) == 42
+
+    bad = client.put('/api/endpoints/enrollment/settings',
+                     json={'maxpendingenrollments': -1}, headers=auth_headers)
+    assert bad.status_code == 400

tests/test_plugins/test_endpoints_fleet_alert_scheduling.py

@@ -82,7 +82,7 @@ def test_fleet_alert_respects_future_starttime(client, db, app):
     db.session.commit()
 
     alerts = _poll(client, app, 'TEST-PC-01')
-    assert [a['notificationId'] for a in alerts] == [note.notificationid]
+    assert [a['notificationid'] for a in alerts] == [note.notificationid]
     assert alerts[0]['title'] == 'Scheduled maintenance at 3:30 PM'
 
 
@@ -91,7 +91,7 @@ def test_fleet_alert_with_no_starttime_appears_immediately(client, db, app):
     note = _fleet_alert(db, 'Immediate alert - urgent')
 
     alerts = _poll(client, app, 'TEST-PC-02')
-    assert [a['notificationId'] for a in alerts] == [note.notificationid]
+    assert [a['notificationid'] for a in alerts] == [note.notificationid]
     assert alerts[0]['title'] == 'Immediate alert - urgent'
 
 
@@ -101,7 +101,7 @@ def test_fleet_alert_past_starttime_appears(client, db, app):
                         starttime=_now() - timedelta(hours=1))
 
     alerts = _poll(client, app, 'TEST-PC-03')
-    assert [a['notificationId'] for a in alerts] == [note.notificationid]
+    assert [a['notificationid'] for a in alerts] == [note.notificationid]
 
 
 def test_fleet_alert_expired_does_not_appear(client, db, app):

tests/test_plugins/test_endpoints_fleet_alerts.py

@@ -107,18 +107,42 @@ def test_agent_polls_its_active_fleet_alerts(client, db, app):
 
     assert resp.status_code == 200, resp.get_data(as_text=True)
     alerts = resp.get_json()['data']
-    assert [a['notificationId'] for a in alerts] == [alert.notificationid]
+    assert [a['notificationid'] for a in alerts] == [alert.notificationid]
     assert alerts[0]['message'] == 'Line 3 is down'
     assert alerts[0]['timeout'] == 60
     # The tray renders the type colour, so it has to come down with the alert
     # rather than being guessed client side.
-    assert alerts[0]['typeColor'] == '#d93f3f'
+    assert alerts[0]['typecolor'] == '#d93f3f'
+
+
+def test_the_feed_still_carries_the_legacy_camelcase_keys(client, db, app):
+    """TRANSITION for trays older than 1.7.0.
+
+    A deployed 1.6.x tray fetches this feed itself and deserializes
+    case-sensitively, so serving only the lowercase keys would make every alert
+    disappear on those PCs with no error anywhere. Both spellings go down, with
+    identical values, until the fleet is upgraded - then this test and the two
+    camelCase keys go together.
+    """
+    _computer(db, 'BAY-01')
+    alert = _fleet_alert(db, message='Line 3 is down')
+
+    resp = client.get(f'{ALERTS_URL}?hostname=BAY-01',
+                      headers=_collector_headers(app))
+
+    row = resp.get_json()['data'][0]
+    assert row['notificationId'] == row['notificationid'] == alert.notificationid
+    assert row['typeColor'] == row['typecolor'] == '#d93f3f'
 
 
 def test_poll_requires_the_collector_key(client, db, app):
     """The alert feed is agent-facing, not anonymous."""
     _computer(db, 'BAY-01')
     _fleet_alert(db)
+    # Configure the key here rather than inheriting it from whichever test ran
+    # first: under xdist this file is split across workers, and an unconfigured
+    # key answers 500 instead of the 401 the test is about.
+    app.config['COLLECTOR_API_KEY'] = COLLECTOR_KEY
 
     resp = client.get(f'{ALERTS_URL}?hostname=BAY-01')
 
@@ -251,7 +275,7 @@ def test_one_bay_cannot_acknowledge_for_another(client, db, app):
     # And BAY-02's alert is untouched by the attempt.
     feed = client.get(f'{ALERTS_URL}?hostname=BAY-02',
                       headers=_collector_headers(app))
-    assert [a['notificationId'] for a in feed.get_json()['data']] == [
+    assert [a['notificationid'] for a in feed.get_json()['data']] == [
         alert.notificationid]
 
 
@@ -267,7 +291,7 @@ def test_acknowledging_one_bay_leaves_the_others_alerted(client, db, app):
                 headers=headers)
 
     still = client.get(f'{ALERTS_URL}?hostname=BAY-02', headers=headers)
-    assert [a['notificationId'] for a in still.get_json()['data']] == [
+    assert [a['notificationid'] for a in still.get_json()['data']] == [
         alert.notificationid]
 
 
@@ -323,7 +347,7 @@ def _ids(client, app, hostname):
     resp = client.get(f'{ALERTS_URL}?hostname={hostname}',
                       headers=_collector_headers(app))
     assert resp.status_code == 200, resp.get_data(as_text=True)
-    return [a['notificationId'] for a in resp.get_json()['data']]
+    return [a['notificationid'] for a in resp.get_json()['data']]
 
 
 def test_no_scope_still_means_the_whole_fleet(client, db, app):