Search spelling suggestions across every list

shopdb/core/services/spellfix.py

@@ -0,0 +1,169 @@
+"""Suggest a spelling for a search word that found nothing.
+
+WHY NOT SOUNDEX. MySQL has it; SQLite does not, and the suite runs on SQLite
+while production runs MySQL - so a SOUNDEX() in a query is either an error in
+every test or a production path no test ever executes. It is also wrong for this
+data: soundex is English-name phonetics, four characters wide, and it DISCARDS
+DIGITS, so CSF16 and CSF17 hash identically. Half of what people search here is
+an identifier.
+
+So this is character-based, runs in Python, and behaves the same on both
+dialects.
+
+THE VOCABULARY IS THE DATA. No dictionary knows Genspect, Telesis, Keyence or
+wax-trace. The terms come from the columns being searched, which also means the
+corpus is never stale: a vendor added this morning is correctable this morning.
+
+IT SUGGESTS, IT DOES NOT REWRITE. The caller offers "did you mean" on a search
+that found nothing. Silently searching for something else is how somebody orders
+the wrong cartridge.
+"""
+
+from collections import defaultdict
+
+# Longest edit distance allowed, by length of the word being corrected. A short
+# word has few ways to be wrong, so one edit is already a different word: 'bay'
+# to 'bar' is not a typo anybody wants guessed for them.
+_DISTANCE_BY_LENGTH = ((4, 1), (8, 2))
+_MAX_DISTANCE = 3
+
+# Below this a word is too short to correct at all.
+MIN_LENGTH = 3
+
+# Candidates sharing no trigram with the query are not scored. Cuts the work
+# from the whole vocabulary to a handful without changing the answer, because a
+# word within the distances above always shares at least one trigram.
+_TRIGRAM = 3
+
+
+def _digits(word):
+    return ''.join(c for c in word if c.isdigit())
+
+
+def max_distance(word):
+    for length, allowed in _DISTANCE_BY_LENGTH:
+        if len(word) <= length:
+            return allowed
+    return _MAX_DISTANCE
+
+
+def damerau_levenshtein(first, second, cutoff=None):
+    """Edit distance counting a transposition as ONE edit, not two.
+
+    Transposition is the commonest typing error - 'Keyecne' for 'Keyence' - and
+    plain Levenshtein charges two for it, which pushes real typos past the
+    threshold on short words.
+
+    `cutoff` abandons a comparison that cannot come in under it. Most candidates
+    are hopeless and this is the difference between scoring a vocabulary and
+    scanning one.
+    """
+    if first == second:
+        return 0
+    if abs(len(first) - len(second)) > (cutoff if cutoff is not None else 99):
+        return (cutoff or 99) + 1
+
+    previous = list(range(len(second) + 1))
+    beforelast = None
+    for i, a in enumerate(first, 1):
+        current = [i] + [0] * len(second)
+        for j, b in enumerate(second, 1):
+            cost = 0 if a == b else 1
+            current[j] = min(previous[j] + 1,        # deletion
+                             current[j - 1] + 1,     # insertion
+                             previous[j - 1] + cost)  # substitution
+            if (beforelast is not None and i > 1 and j > 1
+                    and a == second[j - 2] and first[i - 2] == b):
+                current[j] = min(current[j], beforelast[j - 2] + cost)
+        if cutoff is not None and min(current) > cutoff:
+            return cutoff + 1
+        beforelast, previous = previous, current
+    return previous[-1]
+
+
+def _trigrams(word):
+    padded = '  %s ' % word
+    return {padded[i:i + _TRIGRAM] for i in range(len(padded) - _TRIGRAM + 1)}
+
+
+class Vocabulary:
+    """Words harvested from searchable data, indexed for near-match lookup."""
+
+    def __init__(self, terms=()):
+        self.words = set()
+        self._bytrigram = defaultdict(set)
+        for term in terms:
+            self.add(term)
+
+    def add(self, term):
+        for word in str(term or '').split():
+            word = word.strip().lower()
+            if len(word) < MIN_LENGTH:
+                continue
+            if word in self.words:
+                continue
+            self.words.add(word)
+            for gram in _trigrams(word):
+                self._bytrigram[gram].add(word)
+
+    def candidates(self, word):
+        found = set()
+        for gram in _trigrams(word):
+            found |= self._bytrigram.get(gram, set())
+        return found
+
+    def suggest(self, word):
+        """The best correction for `word`, or None to leave it alone.
+
+        Returns None rather than a poor guess: a wrong suggestion costs more
+        than no suggestion, because somebody acts on it.
+        """
+        word = (word or '').strip().lower()
+        if len(word) < MIN_LENGTH or word in self.words:
+            return None
+
+        limit = max_distance(word)
+        best, bestdistance = None, limit + 1
+        for candidate in self.candidates(word):
+            if not _acceptable(word, candidate):
+                continue
+            distance = damerau_levenshtein(word, candidate, cutoff=limit)
+            if distance < bestdistance:
+                best, bestdistance = candidate, distance
+            elif distance == bestdistance and best is not None:
+                # A tie means two equally good answers and no way to choose.
+                # Offering either one implies a confidence that is not there.
+                best = None
+        return best if bestdistance <= limit else None
+
+    def suggest_query(self, query):
+        """Correct a whole query, or None if nothing in it can be improved."""
+        words = [w for w in (query or '').split() if w]
+        if not words:
+            return None
+        fixed, changed = [], False
+        for word in words:
+            suggestion = self.suggest(word)
+            if suggestion and suggestion != word.lower():
+                fixed.append(suggestion)
+                changed = True
+            else:
+                fixed.append(word)
+        return ' '.join(fixed) if changed else None
+
+
+def _acceptable(word, candidate):
+    """Guards that keep a correction from destroying an identifier.
+
+    DIGITS MUST MATCH EXACTLY. CSF16 to CSF17 is one edit and a catastrophe: it
+    is a different bay, a different machine, a different cartridge. Any term
+    carrying digits is treated as an identifier, and an identifier is either
+    right or not correctable.
+
+    THE FIRST CHARACTER MUST MATCH. Typing errors land in the middle and at the
+    end far more often than on the first key, so requiring it costs almost no
+    recall and removes most of the nonsense.
+    """
+    if _digits(word) != _digits(candidate):
+        return False
+    return bool(word) and bool(candidate) and word[0] == candidate[0]