shopdb-flask/shopdb/core/services/spellfix.py

"""Suggest a spelling for a search word that found nothing.

WHY NOT SOUNDEX. MySQL has it; SQLite does not, and the suite runs on SQLite
while production runs MySQL - so a SOUNDEX() in a query is either an error in
every test or a production path no test ever executes. It is also wrong for this
data: soundex is English-name phonetics, four characters wide, and it DISCARDS
DIGITS, so CSF16 and CSF17 hash identically. Half of what people search here is
an identifier.

So this is character-based, runs in Python, and behaves the same on both
dialects.

THE VOCABULARY IS THE DATA. No dictionary knows Genspect, Telesis, Keyence or
wax-trace. The terms come from the columns being searched, which also means the
corpus is never stale: a vendor added this morning is correctable this morning.

IT SUGGESTS, IT DOES NOT REWRITE. The caller offers "did you mean" on a search
that found nothing. Silently searching for something else is how somebody orders
the wrong cartridge.
"""

from collections import defaultdict

# Longest edit distance allowed, by length of the word being corrected. A short
# word has few ways to be wrong, so one edit is already a different word: 'bay'
# to 'bar' is not a typo anybody wants guessed for them.
_DISTANCE_BY_LENGTH = ((4, 1), (8, 2))
_MAX_DISTANCE = 3

# Below this a word is too short to correct at all.
MIN_LENGTH = 3

# Candidates sharing no trigram with the query are not scored. Cuts the work
# from the whole vocabulary to a handful without changing the answer, because a
# word within the distances above always shares at least one trigram.
_TRIGRAM = 3


def _digits(word):
    return ''.join(c for c in word if c.isdigit())


def max_distance(word):
    for length, allowed in _DISTANCE_BY_LENGTH:
        if len(word) <= length:
            return allowed
    return _MAX_DISTANCE


def damerau_levenshtein(first, second, cutoff=None):
    """Edit distance counting a transposition as ONE edit, not two.

    Transposition is the commonest typing error - 'Keyecne' for 'Keyence' - and
    plain Levenshtein charges two for it, which pushes real typos past the
    threshold on short words.

    `cutoff` abandons a comparison that cannot come in under it. Most candidates
    are hopeless and this is the difference between scoring a vocabulary and
    scanning one.
    """
    if first == second:
        return 0
    if abs(len(first) - len(second)) > (cutoff if cutoff is not None else 99):
        return (cutoff or 99) + 1

    previous = list(range(len(second) + 1))
    beforelast = None
    for i, a in enumerate(first, 1):
        current = [i] + [0] * len(second)
        for j, b in enumerate(second, 1):
            cost = 0 if a == b else 1
            current[j] = min(previous[j] + 1,        # deletion
                             current[j - 1] + 1,     # insertion
                             previous[j - 1] + cost)  # substitution
            if (beforelast is not None and i > 1 and j > 1
                    and a == second[j - 2] and first[i - 2] == b):
                current[j] = min(current[j], beforelast[j - 2] + cost)
        if cutoff is not None and min(current) > cutoff:
            return cutoff + 1
        beforelast, previous = previous, current
    return previous[-1]


def _trigrams(word):
    padded = '  %s ' % word
    return {padded[i:i + _TRIGRAM] for i in range(len(padded) - _TRIGRAM + 1)}


class Vocabulary:
    """Words harvested from searchable data, indexed for near-match lookup."""

    def __init__(self, terms=()):
        self.words = set()
        self._bytrigram = defaultdict(set)
        for term in terms:
            self.add(term)

    def add(self, term):
        for word in str(term or '').split():
            word = word.strip().lower()
            if len(word) < MIN_LENGTH:
                continue
            if word in self.words:
                continue
            self.words.add(word)
            for gram in _trigrams(word):
                self._bytrigram[gram].add(word)

    def candidates(self, word):
        found = set()
        for gram in _trigrams(word):
            found |= self._bytrigram.get(gram, set())
        return found

    def suggest(self, word):
        """The best correction for `word`, or None to leave it alone.

        Returns None rather than a poor guess: a wrong suggestion costs more
        than no suggestion, because somebody acts on it.
        """
        word = (word or '').strip().lower()
        if len(word) < MIN_LENGTH or word in self.words:
            return None

        limit = max_distance(word)
        best, bestdistance = None, limit + 1
        for candidate in self.candidates(word):
            if not _acceptable(word, candidate):
                continue
            distance = damerau_levenshtein(word, candidate, cutoff=limit)
            if distance < bestdistance:
                best, bestdistance = candidate, distance
            elif distance == bestdistance and best is not None:
                # A tie means two equally good answers and no way to choose.
                # Offering either one implies a confidence that is not there.
                best = None
        return best if bestdistance <= limit else None

    def suggest_query(self, query):
        """Correct a whole query, or None if nothing in it can be improved."""
        words = [w for w in (query or '').split() if w]
        if not words:
            return None
        fixed, changed = [], False
        for word in words:
            suggestion = self.suggest(word)
            if suggestion and suggestion != word.lower():
                fixed.append(suggestion)
                changed = True
            else:
                fixed.append(word)
        return ' '.join(fixed) if changed else None


def _acceptable(word, candidate):
    """Guards that keep a correction from destroying an identifier.

    DIGITS MUST MATCH EXACTLY. CSF16 to CSF17 is one edit and a catastrophe: it
    is a different bay, a different machine, a different cartridge. Any term
    carrying digits is treated as an identifier, and an identifier is either
    right or not correctable.

    THE FIRST CHARACTER MUST MATCH. Typing errors land in the middle and at the
    end far more often than on the first key, so requiring it costs almost no
    recall and removes most of the nonsense.
    """
    if _digits(word) != _digits(candidate):
        return False
    return bool(word) and bool(candidate) and word[0] == candidate[0]