harden CSV import reconciliation and preserve saved model selections

Default fuzzy reconcile threshold to 0 and refuse merges when digit runs differ so numbered labels like Bay 311 never bind to Bay 310. Keep edit forms from clearing saved model fields when category filtering hides untyped rows. Includes in-progress computers policy and endpoints command feature work.

shopdb/__init__.py

@@ -65,7 +65,18 @@ from .plugins import plugin_manager
 # machine number in the PC row instead of hand-building a relationships sheet.
 # Core stays ignorant of the column's meaning. Additive name on the import
 # surface, minor bump.
-__contract_version__ = '0.21.0'
+# 0.22.0: added mint_service_token, list_service_tokens and revoke_service_tokens
+# to shopdb.api so a plugin can provision, list, and revoke a single-scope
+# managed service token (an endpoint agent key) without importing core token
+# internals. The endpoints plugin uses them for a UI-managed, regenerable
+# collector.ingest.endpoints key whose revocation is an instant kill switch.
+# Additive names on the surface, minor bump.
+# 0.23.0: added collector_authorized to shopdb.api so a plugin can gate an
+# agent-facing route that lives OUTSIDE the generic collector blueprint (the
+# endpoints command claim/result long-poll callbacks) on the same per-plugin
+# collector key + managed-token check the ingest route uses, without importing
+# core collector internals. Additive name on the surface, minor bump.
+__contract_version__ = '0.23.0'
 
 # Product release version (see ADR-007). The product version and the
 # plugin-contract version above are distinct series with independent

shopdb/api/__init__.py

@@ -75,8 +75,28 @@ from shopdb.utils.authz import require_permission, require_role
 # internals.
 from shopdb.utils.apitoken_auth import (
     service_token_authorized, authorized_service_token,
+    mint_service_token, list_service_tokens, revoke_service_tokens,
 )
 
+# Collector-token authorization for agent-facing plugin routes that live outside
+# the generic collector blueprint (e.g. the endpoints command claim/result
+# callbacks). Reuses the same per-plugin collector key + managed-token check the
+# ingest route uses, so a plugin never imports core collector internals.
+def collector_authorized(pluginname: str):
+    """Authorize an agent-facing route on a plugin's collector token.
+
+    Returns None when the caller presented a valid collector credential for
+    this plugin (the per-plugin env key or a collector.ingest[.<plugin>]-scoped
+    managed token), else the error_response to return. Lets a plugin gate an
+    agent callback on the same key and kill switch as its ingest route without
+    reaching into core collector internals.
+
+    Imported lazily: shopdb.core.api imports back into shopdb.api (send_upload),
+    so a module-level import here would be a circular import at load time.
+    """
+    from shopdb.core.api.collector import _check_collector_auth, _plugin_api_key
+    return _check_collector_auth(_plugin_api_key(pluginname), pluginname)
+
 # Serving an uploaded file safely. A plugin that serves user-supplied bytes must
 # not have to remember the headers that keep an SVG or an HTML file from running
 # as script in this origin - see shopdb/utils/uploads.py.
@@ -313,6 +333,10 @@ __all__ = [
     'require_role',
     'service_token_authorized',
     'authorized_service_token',
+    'mint_service_token',
+    'list_service_tokens',
+    'revoke_service_tokens',
+    'collector_authorized',
     'SupportTeam',
     # Serving uploads
     'send_upload',

shopdb/core/api/collector.py

@@ -39,17 +39,56 @@ def _computer_models():
         return None
 
 
-def _token_is_collector(token):
-    """True when this managed token is a collector service token: scoped for
-    collector.ingest AND its owner actually holds that permission (admins do)."""
+def _collector_scopes_for(pluginname=None):
+    """The scope names that authorize a collector request.
+
+    The generic scope `collector.ingest` authorizes every collector. A plugin
+    MAY also mint a token scoped to `collector.ingest.<pluginname>` so that
+    rotating/revoking it stops only that plugin's collector traffic (endpoints
+    uses this for its regenerable agent kill switch). Both are accepted.
+    """
+    scopes = {COLLECTOR_SCOPE}
+    if pluginname:
+        scopes.add(f'{COLLECTOR_SCOPE}.{pluginname}')
+    return scopes
+
+
+def _env_fallback_disabled(pluginname=None):
+    """True when a plugin has switched OFF the bootstrap env-key fallback.
+
+    Once an operator provisions a UI-managed collector token, the plugin sets
+    its `plugin.<pluginname>.envkeyfallback` setting to false so the old .env
+    key stops authorizing. That turns key revocation into a real kill switch:
+    rotating or revoking the managed token cannot be bypassed by a lingering env
+    key. Absent
+    setting = fallback stays ON (back-compat, so nothing breaks on upgrade).
+    """
+    if not pluginname:
+        return False
+    from shopdb.core.models.setting import Setting
+    key = f'plugin.{pluginname}.envkeyfallback'
+    return Setting.get(key, default=True) is False
+
+
+def _token_is_collector(token, pluginname=None):
+    """True when this managed token is a collector service token: scoped for a
+    collector scope AND its owner actually holds that permission (admins do).
+
+    Accepts the generic `collector.ingest` or, when pluginname is given, the
+    plugin-specific `collector.ingest.<pluginname>` scope.
+    """
     from shopdb.core.models import User
 
     scopelist = token.scopelist
-    if not scopelist or COLLECTOR_SCOPE not in scopelist:
+    if not scopelist:
+        return False
+    allowed = _collector_scopes_for(pluginname)
+    held = [s for s in scopelist if s in allowed]
+    if not held:
         return False
     user = db.session.get(User, token.userid)
     return (user is not None and user.isactive
-            and user.haspermission(COLLECTOR_SCOPE))
+            and any(user.haspermission(s) for s in held))
 
 
 def _collector_managed_token(api_key):
@@ -73,24 +112,28 @@ def _collector_managed_token(api_key):
     return None
 
 
-def _check_collector_auth(expected_key):
+def _check_collector_auth(expected_key, pluginname=None):
     """Authorize a collector request. Returns None when authorized, else the
     error_response to return.
 
     Accepts a collector-scoped managed token (Bearer or X-API-Key) OR the env
     key via X-API-Key (legacy/bootstrap fallback). Env keys stay supported so
     nothing breaks; a valid managed token works even with no env key set.
+
+    When pluginname is given, a token scoped to collector.ingest.<pluginname>
+    also authorizes (in addition to the generic collector.ingest scope).
     """
     from shopdb.utils.apitoken_auth import touch_apitoken_lastused
 
     api_key = request.headers.get('X-API-Key')
 
     token = _collector_managed_token(api_key)
-    if token is not None and _token_is_collector(token):
+    if token is not None and _token_is_collector(token, pluginname):
         touch_apitoken_lastused(token)
         return None
 
-    if expected_key and api_key == expected_key:
+    if expected_key and api_key == expected_key \
+            and not _env_fallback_disabled(pluginname):
         return None
 
     # Fail-closed only on pure server misconfiguration: no env key AND the
@@ -200,7 +243,7 @@ def generic_collect(pluginname):
 
     # Per-plugin env key (fallback) OR a collector-scoped managed token. Header
     # only (querystring fallback dropped, see require_api_key).
-    error = _check_collector_auth(_plugin_api_key(pluginname))
+    error = _check_collector_auth(_plugin_api_key(pluginname), pluginname)
     if error is not None:
         return error
 

shopdb/core/api/settings.py

@@ -879,10 +879,10 @@ def build_default_settings():
         },
         {
             'key': 'asset_import_reconcile_threshold',
-            'value': '0.85',
+            'value': '0',
             'valuetype': 'string',
             'category': 'site',
-            'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.'
+            'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 (the default) disables fuzzy reconciliation - names must match exactly after normalization. Labels whose digit runs differ (e.g. "Bay 310" vs "Bay 311") are never fuzzy-matched. Operating systems and asset rows are never reconciled.'
         },
     ]
 

shopdb/core/services/csvimport.py

@@ -172,8 +172,11 @@ CORPORATE_SUFFIXES = {
 }
 
 # Default similarity a fuzzy match must reach, 0..1. Also the value the settings
-# default is seeded with; 0 disables reconciliation entirely.
-DEFAULT_RECONCILE_THRESHOLD = 0.85
+# default is seeded with; 0 disables reconciliation entirely. Fuzzy matching is
+# opt-in: it ships OFF (0) because a high similarity score is precisely the
+# dangerous case - 'Bay 310' and 'Bay 311' score 0.857 and would silently merge.
+# A site turns it on deliberately once it understands its own vocabulary.
+DEFAULT_RECONCILE_THRESHOLD = 0.0
 
 
 # -- plugin import hooks (inversion of control) ---------------------------------
@@ -259,6 +262,19 @@ def normalize_label(value):
     return ' '.join(words)
 
 
+def digit_runs(value):
+    """Return the ordered list of digit runs in a normalized label.
+
+    'bay 310' -> ['310'], 'precision 3660' -> ['3660'], 'dell' -> []. Two labels
+    that share a fuzzy word shape but differ in any digit run ('bay 310' vs
+    'bay 311') must NOT reconcile: a single differing digit is a small fraction
+    of a long string, so SequenceMatcher scores it above threshold even though
+    the labels name two distinct things. Comparing digit runs exactly closes
+    that gap while still fuzzing the surrounding words.
+    """
+    return re.findall(r'\d+', value or '')
+
+
 def table_for(name):
     table = db.metadata.tables.get(name)
     if table is None:
@@ -451,6 +467,7 @@ class Resolver:
         wanted = normalize_label(value)
         if not wanted:
             return None
+        wanted_digits = digit_runs(wanted)
 
         best_id = None
         best_label = None
@@ -463,6 +480,12 @@ class Resolver:
             if normalized == wanted:
                 score = 1.0
             else:
+                # Never fuzzy-match two labels whose digit runs differ. 'Bay 310'
+                # and 'Bay 311' share every word and score 0.857, but the digits
+                # name distinct things; refuse rather than silently merge. The
+                # normalized-exact tier above (score 1.0) is unaffected.
+                if digit_runs(normalized) != wanted_digits:
+                    continue
                 score = difflib.SequenceMatcher(None, wanted, normalized).ratio()
             if score > best_score:
                 best_score = score

shopdb/plugins/alembic_template.py

@@ -49,9 +49,11 @@ logger = logging.getLogger('alembic.env.plugin')
 PLUGIN_TABLE_OWNERS: dict[str, Iterable[str]] = {
     'backups':       ('backuprevisions',),
     'computers':     ('computertypes', 'computers', 'computerinstalledapps',
-                      'accessprotocols', 'computeraccess'),
+                      'accessprotocols', 'computeraccess',
+                      'configurationpolicies', 'configurationpolicyactions',
+                      'businesspolicies', 'businesspolicyitems'),
     'employees':     ('directoryemployees',),
-    'endpoints':     ('endpointstates', 'endpointevents'),
+    'endpoints':     ('endpointstates', 'endpointevents', 'endpointcommands'),
     'geenforce':     ('manifestscopes', 'manifestentries', 'manifestentrypctypes',
                       'manifestentryhostnames', 'manifestentrymachinenumbers',
                       'manifestinusechecks', 'manifestinusecheckprocesses',

shopdb/utils/apitoken_auth.py

@@ -107,6 +107,82 @@ def authorized_service_token(scope):
     return token
 
 
+def mint_service_token(scope, name, owner_userid, revoke_existing=True):
+    """Mint a managed service token scoped to exactly ONE permission.
+
+    The contract-surface way for a plugin to provision an unattended service
+    credential (an endpoint agent key, a collector key) without importing core
+    token internals. The returned secret is the ONLY time the full secret is
+    available - callers reveal it once and store only the prefix afterwards.
+
+    scope           the single permission the token authorizes (e.g.
+                    collector.ingest.endpoints). The owner must hold it.
+    name            human label shown in the token list UI.
+    owner_userid    the acting admin who owns (and can be audited for) the key.
+    revoke_existing when True (default), every currently-active token scoped to
+                    EXACTLY this one permission is deactivated first, so
+                    regenerating is a hard cutover: the old agent key stops
+                    working the instant the new one is minted.
+
+    Returns (token, secret). Commits.
+    """
+    from shopdb.core.models import User
+
+    owner = db.session.get(User, owner_userid)
+    if owner is None or not owner.isactive or not owner.haspermission(scope):
+        raise PermissionError(
+            'Token owner must be an active user holding ' + scope)
+
+    if revoke_existing:
+        for existing in list_service_tokens(scope):
+            existing.isactive = False
+
+    secret = ApiToken.generate_secret()
+    token = ApiToken(
+        userid=owner_userid,
+        name=name,
+        tokenprefix=ApiToken.prefix_of(secret),
+        tokenhash=ApiToken.hash_secret(secret),
+    )
+    token.scopelist = [scope]
+    db.session.add(token)
+    db.session.commit()
+    return token, secret
+
+
+def list_service_tokens(scope, active_only=True):
+    """Return managed tokens scoped to EXACTLY the single permission `scope`.
+
+    Used by a plugin settings page to render the current agent-key state
+    (prefix, created, last-used) without exposing token internals. Filters in
+    Python because scopes are stored as a JSON blob, not a queryable column.
+    """
+    query = ApiToken.query
+    if active_only:
+        query = query.filter_by(isactive=True)
+    matches = []
+    for token in query.order_by(ApiToken.createddate.desc()).all():
+        scopelist = token.scopelist
+        if scopelist and len(scopelist) == 1 and scopelist[0] == scope:
+            matches.append(token)
+    return matches
+
+
+def revoke_service_tokens(scope):
+    """Deactivate every active managed token scoped to EXACTLY `scope`.
+
+    The kill switch: revoking the endpoint agent key stops all endpoint-agent
+    traffic authorized by a managed token. Returns the count revoked. Commits.
+    """
+    revoked = 0
+    for token in list_service_tokens(scope):
+        token.isactive = False
+        revoked += 1
+    if revoked:
+        db.session.commit()
+    return revoked
+
+
 def service_token_authorized(scope):
     """True when the current request carries a managed token scoped for `scope`
     whose owner is active and holds that permission. Accepts X-API-Key or a