diff --git a/shopdb/core/api/search.py b/shopdb/core/api/search.py index 85c3753..a6c63c1 100644 --- a/shopdb/core/api/search.py +++ b/shopdb/core/api/search.py @@ -1099,6 +1099,16 @@ def global_search(): total_all = len(unique_results) + # A search that found NOTHING is the only evidence of what people cannot + # find, and it is not otherwise recorded anywhere. Logged at INFO with a + # stable prefix so a week of it can be grepped into a list, which is what + # should decide whether spelling correction is worth wiring in - the recent + # multi-word and cross-field matching changes may already have fixed a good + # share of what used to fail. Guessing at that would be building for a + # problem nobody has measured. + if not unique_results: + logger.info('search-no-results query=%r', query) + # Limit total results unique_results = unique_results[:50] diff --git a/shopdb/core/services/spellfix.py b/shopdb/core/services/spellfix.py new file mode 100644 index 0000000..36235f4 --- /dev/null +++ b/shopdb/core/services/spellfix.py @@ -0,0 +1,169 @@ +"""Suggest a spelling for a search word that found nothing. + +WHY NOT SOUNDEX. MySQL has it; SQLite does not, and the suite runs on SQLite +while production runs MySQL - so a SOUNDEX() in a query is either an error in +every test or a production path no test ever executes. It is also wrong for this +data: soundex is English-name phonetics, four characters wide, and it DISCARDS +DIGITS, so CSF16 and CSF17 hash identically. Half of what people search here is +an identifier. + +So this is character-based, runs in Python, and behaves the same on both +dialects. + +THE VOCABULARY IS THE DATA. No dictionary knows Genspect, Telesis, Keyence or +wax-trace. The terms come from the columns being searched, which also means the +corpus is never stale: a vendor added this morning is correctable this morning. + +IT SUGGESTS, IT DOES NOT REWRITE. The caller offers "did you mean" on a search +that found nothing. Silently searching for something else is how somebody orders +the wrong cartridge. +""" + +from collections import defaultdict + +# Longest edit distance allowed, by length of the word being corrected. A short +# word has few ways to be wrong, so one edit is already a different word: 'bay' +# to 'bar' is not a typo anybody wants guessed for them. +_DISTANCE_BY_LENGTH = ((4, 1), (8, 2)) +_MAX_DISTANCE = 3 + +# Below this a word is too short to correct at all. +MIN_LENGTH = 3 + +# Candidates sharing no trigram with the query are not scored. Cuts the work +# from the whole vocabulary to a handful without changing the answer, because a +# word within the distances above always shares at least one trigram. +_TRIGRAM = 3 + + +def _digits(word): + return ''.join(c for c in word if c.isdigit()) + + +def max_distance(word): + for length, allowed in _DISTANCE_BY_LENGTH: + if len(word) <= length: + return allowed + return _MAX_DISTANCE + + +def damerau_levenshtein(first, second, cutoff=None): + """Edit distance counting a transposition as ONE edit, not two. + + Transposition is the commonest typing error - 'Keyecne' for 'Keyence' - and + plain Levenshtein charges two for it, which pushes real typos past the + threshold on short words. + + `cutoff` abandons a comparison that cannot come in under it. Most candidates + are hopeless and this is the difference between scoring a vocabulary and + scanning one. + """ + if first == second: + return 0 + if abs(len(first) - len(second)) > (cutoff if cutoff is not None else 99): + return (cutoff or 99) + 1 + + previous = list(range(len(second) + 1)) + beforelast = None + for i, a in enumerate(first, 1): + current = [i] + [0] * len(second) + for j, b in enumerate(second, 1): + cost = 0 if a == b else 1 + current[j] = min(previous[j] + 1, # deletion + current[j - 1] + 1, # insertion + previous[j - 1] + cost) # substitution + if (beforelast is not None and i > 1 and j > 1 + and a == second[j - 2] and first[i - 2] == b): + current[j] = min(current[j], beforelast[j - 2] + cost) + if cutoff is not None and min(current) > cutoff: + return cutoff + 1 + beforelast, previous = previous, current + return previous[-1] + + +def _trigrams(word): + padded = ' %s ' % word + return {padded[i:i + _TRIGRAM] for i in range(len(padded) - _TRIGRAM + 1)} + + +class Vocabulary: + """Words harvested from searchable data, indexed for near-match lookup.""" + + def __init__(self, terms=()): + self.words = set() + self._bytrigram = defaultdict(set) + for term in terms: + self.add(term) + + def add(self, term): + for word in str(term or '').split(): + word = word.strip().lower() + if len(word) < MIN_LENGTH: + continue + if word in self.words: + continue + self.words.add(word) + for gram in _trigrams(word): + self._bytrigram[gram].add(word) + + def candidates(self, word): + found = set() + for gram in _trigrams(word): + found |= self._bytrigram.get(gram, set()) + return found + + def suggest(self, word): + """The best correction for `word`, or None to leave it alone. + + Returns None rather than a poor guess: a wrong suggestion costs more + than no suggestion, because somebody acts on it. + """ + word = (word or '').strip().lower() + if len(word) < MIN_LENGTH or word in self.words: + return None + + limit = max_distance(word) + best, bestdistance = None, limit + 1 + for candidate in self.candidates(word): + if not _acceptable(word, candidate): + continue + distance = damerau_levenshtein(word, candidate, cutoff=limit) + if distance < bestdistance: + best, bestdistance = candidate, distance + elif distance == bestdistance and best is not None: + # A tie means two equally good answers and no way to choose. + # Offering either one implies a confidence that is not there. + best = None + return best if bestdistance <= limit else None + + def suggest_query(self, query): + """Correct a whole query, or None if nothing in it can be improved.""" + words = [w for w in (query or '').split() if w] + if not words: + return None + fixed, changed = [], False + for word in words: + suggestion = self.suggest(word) + if suggestion and suggestion != word.lower(): + fixed.append(suggestion) + changed = True + else: + fixed.append(word) + return ' '.join(fixed) if changed else None + + +def _acceptable(word, candidate): + """Guards that keep a correction from destroying an identifier. + + DIGITS MUST MATCH EXACTLY. CSF16 to CSF17 is one edit and a catastrophe: it + is a different bay, a different machine, a different cartridge. Any term + carrying digits is treated as an identifier, and an identifier is either + right or not correctable. + + THE FIRST CHARACTER MUST MATCH. Typing errors land in the middle and at the + end far more often than on the first key, so requiring it costs almost no + recall and removes most of the nonsense. + """ + if _digits(word) != _digits(candidate): + return False + return bool(word) and bool(candidate) and word[0] == candidate[0] diff --git a/tests/test_core/test_spellfix.py b/tests/test_core/test_spellfix.py new file mode 100644 index 0000000..faea9b3 --- /dev/null +++ b/tests/test_core/test_spellfix.py @@ -0,0 +1,109 @@ +"""Correcting a misspelled search word without wrecking an identifier. + +Pure arithmetic over a word list, so it is tested that way. The cases that +matter most are the ones where it must REFUSE: a wrong suggestion is acted on, +and here that means the wrong bay or the wrong cartridge. +""" + +from shopdb.core.services.spellfix import ( + Vocabulary, damerau_levenshtein, max_distance, +) + +SHOPFLOOR = [ + 'Keyence vision system', 'Genspect', 'Telesis part marker', + 'wax-trace', 'Zebra ZT411', 'calibration', 'Haas Automation', + 'CSF16', 'CSF17', 'MT-1042', 'FC733SY3', 'printer', 'grinder', +] + + +def vocab(): + return Vocabulary(SHOPFLOOR) + + +# ------------------------------------------------------------------ distance + +def test_a_transposition_costs_one_edit_not_two(): + """The commonest typing error. Plain Levenshtein charges two, which pushes + a real typo past the threshold on a short word.""" + assert damerau_levenshtein('keyecne', 'keyence') == 1 + + +def test_distance_allowed_scales_with_length(): + assert max_distance('bay') == 1 + assert max_distance('grinder') == 2 + assert max_distance('calibration') == 3 + + +# ------------------------------------------------------------------ suggests + +def test_it_corrects_vocabulary_no_dictionary_would_hold(): + assert vocab().suggest('keyecne') == 'keyence' + assert vocab().suggest('genspct') == 'genspect' + assert vocab().suggest('telisis') == 'telesis' + + +def test_a_word_that_is_already_right_is_left_alone(): + assert vocab().suggest('keyence') is None + assert vocab().suggest('printer') is None + + +def test_a_word_nothing_resembles_gets_no_guess(): + """A wrong suggestion costs more than no suggestion.""" + assert vocab().suggest('helicopter') is None + + +# ------------------------------------------------------- identifier guards + +def test_an_asset_number_is_never_corrected_to_a_different_one(): + """THE case this exists for. CSF16 to CSF17 is one edit and a different bay. + + Both are in the vocabulary, so without the digit guard the near miss is + exactly the kind of thing edit distance loves to 'fix'. + """ + assert vocab().suggest('csf18') is None + assert vocab().suggest('mt-1043') is None + + +def test_a_typo_in_the_letters_of_an_identifier_still_corrects(): + """The guard is on the DIGITS, not on identifiers as a whole: the digits + have to match, and here they do.""" + assert vocab().suggest('cfs16') == 'csf16' + + +def test_the_first_character_must_match(): + """Typos land in the middle and at the end. Requiring the first key costs + little recall and removes most of the nonsense.""" + assert vocab().suggest('teyence') is None + + +def test_a_tie_produces_no_suggestion(): + """Two equally good answers means there is no answer. CSF16 and CSF17 are + both one edit from CSF1x, and offering either implies a confidence that is + not there - but the digit guard rejects both first, so build a case where + only the tie is in play.""" + tied = Vocabulary(['barn', 'bard']) + assert tied.suggest('bare') is None + + +# ---------------------------------------------------------------- queries + +def test_a_multi_word_query_corrects_only_what_is_wrong(): + assert vocab().suggest_query('keyecne calibration') == 'keyence calibration' + + +def test_a_query_that_is_entirely_fine_returns_none(): + assert vocab().suggest_query('keyence calibration') is None + + +def test_short_words_are_left_alone(): + """Below three characters there is not enough word to correct.""" + assert vocab().suggest('cs') is None + assert vocab().suggest_query('cs') is None + + +def test_the_vocabulary_comes_from_the_data(): + """A vendor added this morning is correctable this morning.""" + fresh = vocab() + assert fresh.suggest('renishw') is None + fresh.add('Renishaw') + assert fresh.suggest('renishw') == 'renishaw'