harden CSV import reconciliation and preserve saved model selections

Default fuzzy reconcile threshold to 0 and refuse merges when digit runs differ so numbered labels like Bay 311 never bind to Bay 310. Keep edit forms from clearing saved model fields when category filtering hides untyped rows. Includes in-progress computers policy and endpoints command feature work.
This commit is contained in:
benjamin.baker
2026-08-28 09:29:09 -04:00
parent d68c6db9ab
commit 01734e5449
46 changed files with 4241 additions and 52 deletions

View File

@@ -62,7 +62,15 @@ EXEMPT_ENDPOINTS = {'knowledgebase.track_click', 'users.update_user',
# GET urls and a preview must not consume a one-view link.
# CREATING one is permission-gated and IS swept.
# Justification in plugins/tools/README.md.
'tools.redeem_secret'}
'tools.redeem_secret',
# Agent-facing command result callback. Authenticated by the
# endpoints collector API key (same shape as the exempt
# collector blueprint), not JWT+role: the endpoint agent has
# no user account, the collector key is its credential and
# kill switch. Enqueue (WebUI -> operator) IS permission-gated
# on endpoints.command and IS swept. Contract in
# plugins/endpoints/AGENT-CONTRACT.md section 7.
'endpoints.report_command_result'}
@pytest.fixture(autouse=True)

View File

@@ -12,6 +12,7 @@ from shopdb.core.services.csvimport import (
IMPORTABLE, ImportError_, Resolver, generate_template, import_csv,
importable_columns, required_columns,
)
from shopdb.core.services.csvimport import digit_runs, normalize_label
from shopdb.extensions import db as _db
@@ -280,3 +281,57 @@ def test_template_example_set_imports_cleanly(db):
table, [str(p) for p in result.problems])
_db.session.rollback()
# -- fuzzy reconciliation safety (review item 1.1) -----------------------------
#
# A high similarity score is the DANGEROUS case: 'Bay 310' and 'Bay 311' share
# every word and score ~0.857, so a naive threshold silently merges two distinct
# locations. Reconciliation now ships OFF (threshold 0) and, even when a site
# turns it on, refuses any pair whose digit runs differ.
def test_digit_runs_extracts_number_groups():
assert digit_runs(normalize_label('Bay 310')) == ['310']
assert digit_runs(normalize_label('Precision 3660')) == ['3660']
assert digit_runs(normalize_label('Dell Inc.')) == []
def test_reconciliation_is_off_by_default(db):
"""A fresh Resolver does not fuzzy-match: an unknown numbered bay is a row
error, not a silent bind onto its neighbour."""
resolver = Resolver()
seed_reference(resolver)
# 'Bay 3' exists from the reference seed; 'Bay 8' does not.
result = import_csv(
'assets',
'assetnumber,assettypeid,locationid\nA-1,Machine,Bay 8\n',
resolver=resolver)
assert not result.ok
assert not resolver.events
def test_reconcile_refuses_when_digit_runs_differ(db):
"""Even with fuzzy matching turned on, 'Bay 311' must not bind to the
existing 'Bay 3': the digit runs differ, so it is a distinct location."""
resolver = Resolver(reconcile_threshold=0.5)
seed_reference(resolver)
result = import_csv(
'assets',
'assetnumber,assettypeid,locationid\nB-1,Machine,Bay 311\n',
resolver=resolver)
assert not result.ok
assert not resolver.events
def test_reconcile_still_binds_normalized_exact(db):
"""The normalized-exact tier is untouched: 'bay 3' folds to the existing
'Bay 3' regardless of threshold, and is reported as an event."""
resolver = Resolver(reconcile_threshold=0.9)
seed_reference(resolver)
result = import_csv(
'assets',
'assetnumber,assettypeid,locationid\nC-1,Machine,BAY 3\n',
resolver=resolver)
assert result.ok, [str(p) for p in result.problems]
_db.session.rollback()

View File

@@ -47,9 +47,9 @@ CUTOVER_PLUGINS = (
EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS}
EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline'
# computers adds the Entra/Azure device identity columns (azuredeviceid, then
# azureobjectid + azuredevicecategory) on top of its anchor, so its head is not
# the f-string default.
EXPECTED_HEAD_REVISION['computers'] = 'computers0003azurefields'
# azureobjectid + azuredevicecategory), then the Configuration/Business policy
# tables, on top of its anchor, so its head is not the f-string default.
EXPECTED_HEAD_REVISION['computers'] = 'computers0004policies'
# backups is also post-cutover: its 0001 really creates backuprevisions, and
# 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup
# otherwise throws away.
@@ -88,8 +88,10 @@ EXPECTED_HEAD_REVISION['notifications'] = 'notifications0005boardorder'
# warranty adds the proof-of-cover document columns on top of its anchor.
EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof'
# endpoints is a NEW post-cutover plugin: its 0001 genuinely creates the
# endpointstates + endpointevents tables the core chain never owned.
EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0001baseline'
# endpointstates + endpointevents tables the core chain never owned, 0002
# adds the endpointcommands queue plus the MTConnect state columns, and 0003
# adds the generic command target/targetkind columns.
EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0003cmdtarget'
# Plugins built after the cutover: their 0001 baseline really creates tables the
# core chain never owned.