160 lines
6.6 KiB
Python
160 lines
6.6 KiB
Python
"""Conservative, explainable match suggestions for UNRESOLVED values.
|
|
|
|
IMPORT-AGNOSTIC and REPORT-ONLY. This never resolves anything and never writes.
|
|
Given a raw value that the normal rules could not map, it proposes zero or more
|
|
candidate names that already exist in a lookup table, each with a reason and a
|
|
confidence, so a human can accept one into that importer's alias map. Nothing
|
|
here is ever applied automatically - a silent wrong match is the worst failure
|
|
mode in dirty asset data, so every suggestion has to be acknowledged.
|
|
|
|
Two tiers, tried in order, stopping at the first that yields anything:
|
|
|
|
1. Bounded deterministic transforms (confidence 'high'). These are rules that
|
|
fit the shapes actually seen in the data - a bay code with a slash is two
|
|
bays (T6/U6 -> T6, U6), a parenthetical is a note (LD9W(GE90) -> LD9W), a
|
|
decimal is a sub-bay (M6.3 -> M6), a trailing range is its first member
|
|
(L1-4 -> L1). Each transform is a named rule, not a fuzzer, so the reason
|
|
printed next to a suggestion tells a human exactly why it was proposed.
|
|
|
|
2. A single edit-distance-1 fallback (confidence 'low'). Only if no transform
|
|
matched, and only for a one-character difference against an existing name.
|
|
Bounded to distance 1 on purpose: it catches a typo or a dropped letter and
|
|
nothing looser, so it cannot quietly invent a plausible-looking wrong bay.
|
|
|
|
The caller supplies the same normalisation and the lookup it resolves against, so
|
|
suggestions are always checked against what is really in the database.
|
|
"""
|
|
|
|
import re
|
|
from collections import namedtuple
|
|
|
|
from scripts.site_imports.resolver import norm
|
|
|
|
# candidate: the existing name we think the raw value meant.
|
|
# reason: which rule proposed it (shown to the human).
|
|
# confidence: 'high' for a deterministic transform, 'low' for edit-distance.
|
|
Suggestion = namedtuple('Suggestion', ('candidate', 'reason', 'confidence'))
|
|
|
|
_PAREN = re.compile(r'\s*\([^)]*\)')
|
|
|
|
|
|
def _candidates_from_transforms(stripped):
|
|
"""Yield (transformed_name, reason) from the bounded deterministic rules.
|
|
|
|
Operates on the value AFTER the importer has removed its own prefix (e.g.
|
|
WM-). Each rule yields plain names; the caller checks each against the live
|
|
lookup, so a rule that produces a non-existent name simply contributes no
|
|
suggestion.
|
|
"""
|
|
# A parenthetical note: LD9W(GE90) -> LD9W
|
|
if '(' in stripped:
|
|
yield (_PAREN.sub('', stripped).strip(), 'dropped parenthetical note')
|
|
|
|
# A slash means two bays: T6/U6 -> T6 and U6. A bare second half (9/10)
|
|
# inherits the first half's letter prefix (T9/10 -> T9, T10).
|
|
if '/' in stripped:
|
|
left, right = (part.strip() for part in stripped.split('/', 1))
|
|
if left:
|
|
yield (left, 'first of a slash-joined pair')
|
|
if right:
|
|
yield (right, 'second of a slash-joined pair')
|
|
prefix = re.match(r'^([A-Za-z]+)', left)
|
|
if prefix and re.match(r'^\d', right):
|
|
yield (prefix.group(1) + right,
|
|
'second of a slash-joined pair, prefix carried')
|
|
|
|
# A decimal sub-bay: M6.3 -> M6
|
|
if '.' in stripped:
|
|
yield (stripped.split('.', 1)[0].strip(), 'dropped decimal sub-bay')
|
|
|
|
# A trailing numeric range: L1-4 -> L1
|
|
dash = re.match(r'^([A-Za-z]*\d+)-\d+$', stripped)
|
|
if dash:
|
|
yield (dash.group(1), 'first of a numeric range')
|
|
|
|
|
|
def _edit_distance_at_most_one(left, right):
|
|
"""True if `left` and `right` differ by at most one insert/delete/replace.
|
|
|
|
Bounded on purpose. A length gap over one is an immediate no; otherwise walk
|
|
both once and allow a single mismatch. Cheap and strict - it is a typo
|
|
catcher, not a similarity search.
|
|
"""
|
|
if left == right:
|
|
return True
|
|
length_left, length_right = len(left), len(right)
|
|
if abs(length_left - length_right) > 1:
|
|
return False
|
|
if length_left == length_right:
|
|
return sum(a != b for a, b in zip(left, right)) == 1
|
|
# Ensure left is the shorter one, then try to match with one insertion.
|
|
if length_left > length_right:
|
|
left, right = right, left
|
|
length_left, length_right = length_right, length_left
|
|
index_left = index_right = 0
|
|
skipped = False
|
|
while index_left < length_left and index_right < length_right:
|
|
if left[index_left] == right[index_right]:
|
|
index_left += 1
|
|
index_right += 1
|
|
elif skipped:
|
|
return False
|
|
else:
|
|
skipped = True
|
|
index_right += 1
|
|
return True
|
|
|
|
|
|
# A low-confidence edit-distance pass can match many short bay codes at once,
|
|
# which is noise, not help. Cap how many it emits so a human sees the few most
|
|
# plausible typo-fixes and nothing drowns the high-confidence transforms.
|
|
_MAX_LOW_CONFIDENCE = 3
|
|
|
|
|
|
def suggest(raw_value, stripped_value, lookup_by_name, display=None):
|
|
"""Suggestions for one unresolved value. Empty list means "no idea".
|
|
|
|
raw_value the original export string (for reporting only).
|
|
stripped_value the value after the importer removed its own prefix; this is
|
|
what the transforms and the edit-distance search operate on.
|
|
lookup_by_name norm(name) -> id, the live table to match against.
|
|
display optional norm(name) -> original-cased name, so a suggestion
|
|
echoes the database's spelling instead of the normalised key.
|
|
|
|
Deterministic transforms first; only if none of them hits an existing name
|
|
does the edit-distance-1 fallback run, and that fallback is capped. High-
|
|
confidence suggestions are never diluted by low-confidence ones.
|
|
"""
|
|
display = display or {}
|
|
|
|
def spell(key):
|
|
return display.get(key, key)
|
|
|
|
results = []
|
|
seen = set()
|
|
for candidate, reason in _candidates_from_transforms(stripped_value):
|
|
candidate = (candidate or '').strip()
|
|
if not candidate:
|
|
continue
|
|
key = norm(candidate)
|
|
if key in lookup_by_name and key not in seen:
|
|
seen.add(key)
|
|
results.append(Suggestion(spell(key), reason, 'high'))
|
|
if results:
|
|
return results
|
|
|
|
target = norm(stripped_value)
|
|
if not target:
|
|
return results
|
|
low = []
|
|
for existing_key in lookup_by_name:
|
|
if existing_key not in seen and _edit_distance_at_most_one(
|
|
target, existing_key):
|
|
seen.add(existing_key)
|
|
low.append(Suggestion(spell(existing_key),
|
|
'one-character difference', 'low'))
|
|
# Only surface the capped set; too many one-off matches is noise.
|
|
if len(low) <= _MAX_LOW_CONFIDENCE:
|
|
results.extend(sorted(low))
|
|
return results
|