harden CSV import reconciliation and preserve saved model selections

Default fuzzy reconcile threshold to 0 and refuse merges when digit runs differ so numbered labels like Bay 311 never bind to Bay 310. Keep edit forms from clearing saved model fields when category filtering hides untyped rows. Includes in-progress computers policy and endpoints command feature work.
This commit is contained in:
benjamin.baker
2026-08-28 09:29:09 -04:00
parent d68c6db9ab
commit 01734e5449
46 changed files with 4241 additions and 52 deletions

View File

@@ -172,8 +172,11 @@ CORPORATE_SUFFIXES = {
}
# Default similarity a fuzzy match must reach, 0..1. Also the value the settings
# default is seeded with; 0 disables reconciliation entirely.
DEFAULT_RECONCILE_THRESHOLD = 0.85
# default is seeded with; 0 disables reconciliation entirely. Fuzzy matching is
# opt-in: it ships OFF (0) because a high similarity score is precisely the
# dangerous case - 'Bay 310' and 'Bay 311' score 0.857 and would silently merge.
# A site turns it on deliberately once it understands its own vocabulary.
DEFAULT_RECONCILE_THRESHOLD = 0.0
# -- plugin import hooks (inversion of control) ---------------------------------
@@ -259,6 +262,19 @@ def normalize_label(value):
return ' '.join(words)
def digit_runs(value):
"""Return the ordered list of digit runs in a normalized label.
'bay 310' -> ['310'], 'precision 3660' -> ['3660'], 'dell' -> []. Two labels
that share a fuzzy word shape but differ in any digit run ('bay 310' vs
'bay 311') must NOT reconcile: a single differing digit is a small fraction
of a long string, so SequenceMatcher scores it above threshold even though
the labels name two distinct things. Comparing digit runs exactly closes
that gap while still fuzzing the surrounding words.
"""
return re.findall(r'\d+', value or '')
def table_for(name):
table = db.metadata.tables.get(name)
if table is None:
@@ -451,6 +467,7 @@ class Resolver:
wanted = normalize_label(value)
if not wanted:
return None
wanted_digits = digit_runs(wanted)
best_id = None
best_label = None
@@ -463,6 +480,12 @@ class Resolver:
if normalized == wanted:
score = 1.0
else:
# Never fuzzy-match two labels whose digit runs differ. 'Bay 310'
# and 'Bay 311' share every word and score 0.857, but the digits
# name distinct things; refuse rather than silently merge. The
# normalized-exact tier above (score 1.0) is unaffected.
if digit_runs(normalized) != wanted_digits:
continue
score = difflib.SequenceMatcher(None, wanted, normalized).ratio()
if score > best_score:
best_score = score