170 lines
6.3 KiB
Python
170 lines
6.3 KiB
Python
"""Suggest a spelling for a search word that found nothing.
|
|
|
|
WHY NOT SOUNDEX. MySQL has it; SQLite does not, and the suite runs on SQLite
|
|
while production runs MySQL - so a SOUNDEX() in a query is either an error in
|
|
every test or a production path no test ever executes. It is also wrong for this
|
|
data: soundex is English-name phonetics, four characters wide, and it DISCARDS
|
|
DIGITS, so CSF16 and CSF17 hash identically. Half of what people search here is
|
|
an identifier.
|
|
|
|
So this is character-based, runs in Python, and behaves the same on both
|
|
dialects.
|
|
|
|
THE VOCABULARY IS THE DATA. No dictionary knows Genspect, Telesis, Keyence or
|
|
wax-trace. The terms come from the columns being searched, which also means the
|
|
corpus is never stale: a vendor added this morning is correctable this morning.
|
|
|
|
IT SUGGESTS, IT DOES NOT REWRITE. The caller offers "did you mean" on a search
|
|
that found nothing. Silently searching for something else is how somebody orders
|
|
the wrong cartridge.
|
|
"""
|
|
|
|
from collections import defaultdict
|
|
|
|
# Longest edit distance allowed, by length of the word being corrected. A short
|
|
# word has few ways to be wrong, so one edit is already a different word: 'bay'
|
|
# to 'bar' is not a typo anybody wants guessed for them.
|
|
_DISTANCE_BY_LENGTH = ((4, 1), (8, 2))
|
|
_MAX_DISTANCE = 3
|
|
|
|
# Below this a word is too short to correct at all.
|
|
MIN_LENGTH = 3
|
|
|
|
# Candidates sharing no trigram with the query are not scored. Cuts the work
|
|
# from the whole vocabulary to a handful without changing the answer, because a
|
|
# word within the distances above always shares at least one trigram.
|
|
_TRIGRAM = 3
|
|
|
|
|
|
def _digits(word):
|
|
return ''.join(c for c in word if c.isdigit())
|
|
|
|
|
|
def max_distance(word):
|
|
for length, allowed in _DISTANCE_BY_LENGTH:
|
|
if len(word) <= length:
|
|
return allowed
|
|
return _MAX_DISTANCE
|
|
|
|
|
|
def damerau_levenshtein(first, second, cutoff=None):
|
|
"""Edit distance counting a transposition as ONE edit, not two.
|
|
|
|
Transposition is the commonest typing error - 'Keyecne' for 'Keyence' - and
|
|
plain Levenshtein charges two for it, which pushes real typos past the
|
|
threshold on short words.
|
|
|
|
`cutoff` abandons a comparison that cannot come in under it. Most candidates
|
|
are hopeless and this is the difference between scoring a vocabulary and
|
|
scanning one.
|
|
"""
|
|
if first == second:
|
|
return 0
|
|
if abs(len(first) - len(second)) > (cutoff if cutoff is not None else 99):
|
|
return (cutoff or 99) + 1
|
|
|
|
previous = list(range(len(second) + 1))
|
|
beforelast = None
|
|
for i, a in enumerate(first, 1):
|
|
current = [i] + [0] * len(second)
|
|
for j, b in enumerate(second, 1):
|
|
cost = 0 if a == b else 1
|
|
current[j] = min(previous[j] + 1, # deletion
|
|
current[j - 1] + 1, # insertion
|
|
previous[j - 1] + cost) # substitution
|
|
if (beforelast is not None and i > 1 and j > 1
|
|
and a == second[j - 2] and first[i - 2] == b):
|
|
current[j] = min(current[j], beforelast[j - 2] + cost)
|
|
if cutoff is not None and min(current) > cutoff:
|
|
return cutoff + 1
|
|
beforelast, previous = previous, current
|
|
return previous[-1]
|
|
|
|
|
|
def _trigrams(word):
|
|
padded = ' %s ' % word
|
|
return {padded[i:i + _TRIGRAM] for i in range(len(padded) - _TRIGRAM + 1)}
|
|
|
|
|
|
class Vocabulary:
|
|
"""Words harvested from searchable data, indexed for near-match lookup."""
|
|
|
|
def __init__(self, terms=()):
|
|
self.words = set()
|
|
self._bytrigram = defaultdict(set)
|
|
for term in terms:
|
|
self.add(term)
|
|
|
|
def add(self, term):
|
|
for word in str(term or '').split():
|
|
word = word.strip().lower()
|
|
if len(word) < MIN_LENGTH:
|
|
continue
|
|
if word in self.words:
|
|
continue
|
|
self.words.add(word)
|
|
for gram in _trigrams(word):
|
|
self._bytrigram[gram].add(word)
|
|
|
|
def candidates(self, word):
|
|
found = set()
|
|
for gram in _trigrams(word):
|
|
found |= self._bytrigram.get(gram, set())
|
|
return found
|
|
|
|
def suggest(self, word):
|
|
"""The best correction for `word`, or None to leave it alone.
|
|
|
|
Returns None rather than a poor guess: a wrong suggestion costs more
|
|
than no suggestion, because somebody acts on it.
|
|
"""
|
|
word = (word or '').strip().lower()
|
|
if len(word) < MIN_LENGTH or word in self.words:
|
|
return None
|
|
|
|
limit = max_distance(word)
|
|
best, bestdistance = None, limit + 1
|
|
for candidate in self.candidates(word):
|
|
if not _acceptable(word, candidate):
|
|
continue
|
|
distance = damerau_levenshtein(word, candidate, cutoff=limit)
|
|
if distance < bestdistance:
|
|
best, bestdistance = candidate, distance
|
|
elif distance == bestdistance and best is not None:
|
|
# A tie means two equally good answers and no way to choose.
|
|
# Offering either one implies a confidence that is not there.
|
|
best = None
|
|
return best if bestdistance <= limit else None
|
|
|
|
def suggest_query(self, query):
|
|
"""Correct a whole query, or None if nothing in it can be improved."""
|
|
words = [w for w in (query or '').split() if w]
|
|
if not words:
|
|
return None
|
|
fixed, changed = [], False
|
|
for word in words:
|
|
suggestion = self.suggest(word)
|
|
if suggestion and suggestion != word.lower():
|
|
fixed.append(suggestion)
|
|
changed = True
|
|
else:
|
|
fixed.append(word)
|
|
return ' '.join(fixed) if changed else None
|
|
|
|
|
|
def _acceptable(word, candidate):
|
|
"""Guards that keep a correction from destroying an identifier.
|
|
|
|
DIGITS MUST MATCH EXACTLY. CSF16 to CSF17 is one edit and a catastrophe: it
|
|
is a different bay, a different machine, a different cartridge. Any term
|
|
carrying digits is treated as an identifier, and an identifier is either
|
|
right or not correctable.
|
|
|
|
THE FIRST CHARACTER MUST MATCH. Typing errors land in the middle and at the
|
|
end far more often than on the first key, so requiring it costs almost no
|
|
recall and removes most of the nonsense.
|
|
"""
|
|
if _digits(word) != _digits(candidate):
|
|
return False
|
|
return bool(word) and bool(candidate) and word[0] == candidate[0]
|