From fa0f6d7ebc7956b6e0102a00905054aab55e7099 Mon Sep 17 00:00:00 2001 From: cproudlock Date: Fri, 21 Aug 2026 10:36:09 -0400 Subject: [PATCH] Record what nobody can find, and a spelling corrector that refuses to guess Two halves of one question - how to handle a misspelled search - deliberately kept separate, because only one of them should be turned on today. WHAT NOBODY CAN FIND IS NOT RECORDED ANYWHERE. A search returning zero results is the only evidence of the gap between what people look for and what is there, and it vanished. Logged now at INFO with a stable prefix, so a week of it greps into a list. That list is what should decide whether correction is worth wiring in: multi-word matching and cross-field matching both changed in the last day, so a good share of what used to fail may already be found. The failures left over might be typos, or vocabulary nobody has entered, or records that genuinely do not exist - and each wants a different answer. THE CORRECTOR IS BUILT AND NOT WIRED IN. shopdb/core/services/spellfix.py, with tests, ready to attach in about ten lines once there is evidence about what to attach it to. NOT SOUNDEX, which was the obvious candidate. MySQL has it and SQLite does not, and the suite runs on SQLite while production runs MySQL - so a SOUNDEX() in a query is either an error in every test or a production path no test executes. It is also wrong for this data: soundex is English-name phonetics, four characters wide, and it DISCARDS DIGITS, so CSF16 and CSF17 hash identically. Half of what people search here is an identifier. So: character distance in Python, same behaviour on both dialects. THE VOCABULARY IS THE DATA. No dictionary holds Genspect, Telesis, Keyence or wax-trace. Terms come from the columns being searched, which also means a vendor added this morning is correctable this morning. TWO GUARDS, and they are the point rather than a detail. Digits must match EXACTLY: CSF16 to CSF17 is one edit and a different bay, so anything carrying digits is either right or not correctable - while cfs16 to csf16 still works, because the guard is on the digits and not on identifiers wholesale. And the first character must match, since typos land mid-word far more often than on the first key, which costs almost no recall. Distance is Damerau-Levenshtein so a transposition costs one edit rather than two - Keyecne for Keyence is the commonest error there is, and plain Levenshtein pushes it past the threshold on short words. Allowed distance scales with length. A tie returns NOTHING: two equally good candidates means there is no answer, and offering either implies a confidence that is not there. It suggests; it never rewrites. Silently searching for something else is how somebody orders the wrong cartridge. 13 tests, weighted toward the refusals, because those are the cases where being wrong costs something. --- shopdb/core/api/search.py | 10 ++ shopdb/core/services/spellfix.py | 169 +++++++++++++++++++++++++++++++ tests/test_core/test_spellfix.py | 109 ++++++++++++++++++++ 3 files changed, 288 insertions(+) create mode 100644 shopdb/core/services/spellfix.py create mode 100644 tests/test_core/test_spellfix.py diff --git a/shopdb/core/api/search.py b/shopdb/core/api/search.py index 85c3753..a6c63c1 100644 --- a/shopdb/core/api/search.py +++ b/shopdb/core/api/search.py @@ -1099,6 +1099,16 @@ def global_search(): total_all = len(unique_results) + # A search that found NOTHING is the only evidence of what people cannot + # find, and it is not otherwise recorded anywhere. Logged at INFO with a + # stable prefix so a week of it can be grepped into a list, which is what + # should decide whether spelling correction is worth wiring in - the recent + # multi-word and cross-field matching changes may already have fixed a good + # share of what used to fail. Guessing at that would be building for a + # problem nobody has measured. + if not unique_results: + logger.info('search-no-results query=%r', query) + # Limit total results unique_results = unique_results[:50] diff --git a/shopdb/core/services/spellfix.py b/shopdb/core/services/spellfix.py new file mode 100644 index 0000000..36235f4 --- /dev/null +++ b/shopdb/core/services/spellfix.py @@ -0,0 +1,169 @@ +"""Suggest a spelling for a search word that found nothing. + +WHY NOT SOUNDEX. MySQL has it; SQLite does not, and the suite runs on SQLite +while production runs MySQL - so a SOUNDEX() in a query is either an error in +every test or a production path no test ever executes. It is also wrong for this +data: soundex is English-name phonetics, four characters wide, and it DISCARDS +DIGITS, so CSF16 and CSF17 hash identically. Half of what people search here is +an identifier. + +So this is character-based, runs in Python, and behaves the same on both +dialects. + +THE VOCABULARY IS THE DATA. No dictionary knows Genspect, Telesis, Keyence or +wax-trace. The terms come from the columns being searched, which also means the +corpus is never stale: a vendor added this morning is correctable this morning. + +IT SUGGESTS, IT DOES NOT REWRITE. The caller offers "did you mean" on a search +that found nothing. Silently searching for something else is how somebody orders +the wrong cartridge. +""" + +from collections import defaultdict + +# Longest edit distance allowed, by length of the word being corrected. A short +# word has few ways to be wrong, so one edit is already a different word: 'bay' +# to 'bar' is not a typo anybody wants guessed for them. +_DISTANCE_BY_LENGTH = ((4, 1), (8, 2)) +_MAX_DISTANCE = 3 + +# Below this a word is too short to correct at all. +MIN_LENGTH = 3 + +# Candidates sharing no trigram with the query are not scored. Cuts the work +# from the whole vocabulary to a handful without changing the answer, because a +# word within the distances above always shares at least one trigram. +_TRIGRAM = 3 + + +def _digits(word): + return ''.join(c for c in word if c.isdigit()) + + +def max_distance(word): + for length, allowed in _DISTANCE_BY_LENGTH: + if len(word) <= length: + return allowed + return _MAX_DISTANCE + + +def damerau_levenshtein(first, second, cutoff=None): + """Edit distance counting a transposition as ONE edit, not two. + + Transposition is the commonest typing error - 'Keyecne' for 'Keyence' - and + plain Levenshtein charges two for it, which pushes real typos past the + threshold on short words. + + `cutoff` abandons a comparison that cannot come in under it. Most candidates + are hopeless and this is the difference between scoring a vocabulary and + scanning one. + """ + if first == second: + return 0 + if abs(len(first) - len(second)) > (cutoff if cutoff is not None else 99): + return (cutoff or 99) + 1 + + previous = list(range(len(second) + 1)) + beforelast = None + for i, a in enumerate(first, 1): + current = [i] + [0] * len(second) + for j, b in enumerate(second, 1): + cost = 0 if a == b else 1 + current[j] = min(previous[j] + 1, # deletion + current[j - 1] + 1, # insertion + previous[j - 1] + cost) # substitution + if (beforelast is not None and i > 1 and j > 1 + and a == second[j - 2] and first[i - 2] == b): + current[j] = min(current[j], beforelast[j - 2] + cost) + if cutoff is not None and min(current) > cutoff: + return cutoff + 1 + beforelast, previous = previous, current + return previous[-1] + + +def _trigrams(word): + padded = ' %s ' % word + return {padded[i:i + _TRIGRAM] for i in range(len(padded) - _TRIGRAM + 1)} + + +class Vocabulary: + """Words harvested from searchable data, indexed for near-match lookup.""" + + def __init__(self, terms=()): + self.words = set() + self._bytrigram = defaultdict(set) + for term in terms: + self.add(term) + + def add(self, term): + for word in str(term or '').split(): + word = word.strip().lower() + if len(word) < MIN_LENGTH: + continue + if word in self.words: + continue + self.words.add(word) + for gram in _trigrams(word): + self._bytrigram[gram].add(word) + + def candidates(self, word): + found = set() + for gram in _trigrams(word): + found |= self._bytrigram.get(gram, set()) + return found + + def suggest(self, word): + """The best correction for `word`, or None to leave it alone. + + Returns None rather than a poor guess: a wrong suggestion costs more + than no suggestion, because somebody acts on it. + """ + word = (word or '').strip().lower() + if len(word) < MIN_LENGTH or word in self.words: + return None + + limit = max_distance(word) + best, bestdistance = None, limit + 1 + for candidate in self.candidates(word): + if not _acceptable(word, candidate): + continue + distance = damerau_levenshtein(word, candidate, cutoff=limit) + if distance < bestdistance: + best, bestdistance = candidate, distance + elif distance == bestdistance and best is not None: + # A tie means two equally good answers and no way to choose. + # Offering either one implies a confidence that is not there. + best = None + return best if bestdistance <= limit else None + + def suggest_query(self, query): + """Correct a whole query, or None if nothing in it can be improved.""" + words = [w for w in (query or '').split() if w] + if not words: + return None + fixed, changed = [], False + for word in words: + suggestion = self.suggest(word) + if suggestion and suggestion != word.lower(): + fixed.append(suggestion) + changed = True + else: + fixed.append(word) + return ' '.join(fixed) if changed else None + + +def _acceptable(word, candidate): + """Guards that keep a correction from destroying an identifier. + + DIGITS MUST MATCH EXACTLY. CSF16 to CSF17 is one edit and a catastrophe: it + is a different bay, a different machine, a different cartridge. Any term + carrying digits is treated as an identifier, and an identifier is either + right or not correctable. + + THE FIRST CHARACTER MUST MATCH. Typing errors land in the middle and at the + end far more often than on the first key, so requiring it costs almost no + recall and removes most of the nonsense. + """ + if _digits(word) != _digits(candidate): + return False + return bool(word) and bool(candidate) and word[0] == candidate[0] diff --git a/tests/test_core/test_spellfix.py b/tests/test_core/test_spellfix.py new file mode 100644 index 0000000..faea9b3 --- /dev/null +++ b/tests/test_core/test_spellfix.py @@ -0,0 +1,109 @@ +"""Correcting a misspelled search word without wrecking an identifier. + +Pure arithmetic over a word list, so it is tested that way. The cases that +matter most are the ones where it must REFUSE: a wrong suggestion is acted on, +and here that means the wrong bay or the wrong cartridge. +""" + +from shopdb.core.services.spellfix import ( + Vocabulary, damerau_levenshtein, max_distance, +) + +SHOPFLOOR = [ + 'Keyence vision system', 'Genspect', 'Telesis part marker', + 'wax-trace', 'Zebra ZT411', 'calibration', 'Haas Automation', + 'CSF16', 'CSF17', 'MT-1042', 'FC733SY3', 'printer', 'grinder', +] + + +def vocab(): + return Vocabulary(SHOPFLOOR) + + +# ------------------------------------------------------------------ distance + +def test_a_transposition_costs_one_edit_not_two(): + """The commonest typing error. Plain Levenshtein charges two, which pushes + a real typo past the threshold on a short word.""" + assert damerau_levenshtein('keyecne', 'keyence') == 1 + + +def test_distance_allowed_scales_with_length(): + assert max_distance('bay') == 1 + assert max_distance('grinder') == 2 + assert max_distance('calibration') == 3 + + +# ------------------------------------------------------------------ suggests + +def test_it_corrects_vocabulary_no_dictionary_would_hold(): + assert vocab().suggest('keyecne') == 'keyence' + assert vocab().suggest('genspct') == 'genspect' + assert vocab().suggest('telisis') == 'telesis' + + +def test_a_word_that_is_already_right_is_left_alone(): + assert vocab().suggest('keyence') is None + assert vocab().suggest('printer') is None + + +def test_a_word_nothing_resembles_gets_no_guess(): + """A wrong suggestion costs more than no suggestion.""" + assert vocab().suggest('helicopter') is None + + +# ------------------------------------------------------- identifier guards + +def test_an_asset_number_is_never_corrected_to_a_different_one(): + """THE case this exists for. CSF16 to CSF17 is one edit and a different bay. + + Both are in the vocabulary, so without the digit guard the near miss is + exactly the kind of thing edit distance loves to 'fix'. + """ + assert vocab().suggest('csf18') is None + assert vocab().suggest('mt-1043') is None + + +def test_a_typo_in_the_letters_of_an_identifier_still_corrects(): + """The guard is on the DIGITS, not on identifiers as a whole: the digits + have to match, and here they do.""" + assert vocab().suggest('cfs16') == 'csf16' + + +def test_the_first_character_must_match(): + """Typos land in the middle and at the end. Requiring the first key costs + little recall and removes most of the nonsense.""" + assert vocab().suggest('teyence') is None + + +def test_a_tie_produces_no_suggestion(): + """Two equally good answers means there is no answer. CSF16 and CSF17 are + both one edit from CSF1x, and offering either implies a confidence that is + not there - but the digit guard rejects both first, so build a case where + only the tie is in play.""" + tied = Vocabulary(['barn', 'bard']) + assert tied.suggest('bare') is None + + +# ---------------------------------------------------------------- queries + +def test_a_multi_word_query_corrects_only_what_is_wrong(): + assert vocab().suggest_query('keyecne calibration') == 'keyence calibration' + + +def test_a_query_that_is_entirely_fine_returns_none(): + assert vocab().suggest_query('keyence calibration') is None + + +def test_short_words_are_left_alone(): + """Below three characters there is not enough word to correct.""" + assert vocab().suggest('cs') is None + assert vocab().suggest_query('cs') is None + + +def test_the_vocabulary_comes_from_the_data(): + """A vendor added this morning is correctable this morning.""" + fresh = vocab() + assert fresh.suggest('renishw') is None + fresh.add('Renishaw') + assert fresh.suggest('renishw') == 'renishaw'