Offer a spelling when a search finds nothing

The corrector landed in fa0f6d7 unwired, pending a week of zero-result logs to
say what it should be aimed at. Attaching it now instead: the guards that make
it safe here are principled rather than tuned to any particular data, so the
logs would confirm the shape of the problem without changing the answer.

ONLY ON ZERO RESULTS. A search that finds something pays for none of this - no
vocabulary build, no scoring, no extra query.

The vocabulary is harvested from columns people actually search - asset numbers
and names, vendors, models, applications, asset types - cached ten minutes and
capped per column. Long enough that someone mistyping twice does not rebuild it,
short enough that a vendor entered this morning is correctable before lunch. The
cap is there because this exists to catch a typo against the names in use, not
to mirror the database into memory to answer one query.

SUGGESTED, NEVER APPLIED. The response carries the better spelling and the
caller decides. Searching for something else on a person's behalf is how
somebody orders the wrong cartridge.

FAILS SOFT. A missing column on a lean install, or anything else, returns no
suggestion and the search answers normally. No vocabulary is worse than the
alternative; it is not fatal.

Three tests through the endpoint rather than only the unit: a misspelling gets a
suggestion, a search that succeeds gets none, and CSF18 against an existing
CSF17 returns no results AND no suggestion - the digit guard holding across the
whole stack, which is the one that would cost somebody a wrong bay.

The zero-result logging stays. It answers "what can people not find", which the
corrector does not: a typo is only one of the reasons a search comes back empty,
and the others are vocabulary nobody has entered and records that do not exist.

NOT DONE: the search page does not render the suggestion yet, so this is
invisible to a person until that Vue change lands.

shopdb/core/api/search.py

@@ -9,13 +9,14 @@ from flask import Blueprint, request, current_app
 from flask_jwt_extended import jwt_required
 from sqlalchemy.orm import joinedload
 
-from shopdb.extensions import db
+from shopdb.extensions import db, cache
 from shopdb.core.models import (
     Application, Setting,
     Asset, AssetType, Communication, Vendor, Model,
     CustomField, CustomFieldValue
 )
 from shopdb.core.api.settings import get_cached_settings
+from shopdb.core.services.spellfix import Vocabulary
 from shopdb.utils.responses import success_response
 
 logger = logging.getLogger(__name__)
@@ -1003,6 +1004,70 @@ def _check_smart_redirect(query, classification):
     return None
 
 
+# How long a harvested vocabulary is reused. Long enough that a burst of
+# searching does not rebuild it, short enough that something entered this
+# morning is correctable before lunch.
+_VOCABULARY_TTL = 600
+
+# Per-column ceiling on harvested terms. A correction is only offered when a
+# search found NOTHING, so the vocabulary exists to catch a typo against the
+# names people actually use - not to mirror the database. Without a bound this
+# would load every asset number on a site into memory to answer one query.
+_VOCABULARY_LIMIT = 5000
+
+
+def _search_vocabulary():
+    """Words worth correcting a search against, harvested from the data.
+
+    From the DATA and not a dictionary, because no dictionary holds Genspect,
+    Telesis, Keyence or wax-trace - and because a vendor added this morning is
+    then correctable this morning.
+
+    Cached: this runs only on a search that found nothing, but a person who
+    mistypes once usually mistypes twice.
+    """
+    cached = cache.get('search_vocabulary')
+    if cached is not None:
+        return cached
+
+    vocabulary = Vocabulary()
+    columns = (
+        (Asset, (Asset.assetnumber, Asset.name)),
+        (Vendor, (Vendor.vendor,)),
+        (Model, (Model.modelnumber,)),
+        (Application, (Application.appname,)),
+        (AssetType, (AssetType.assettype,)),
+    )
+    for model, fields in columns:
+        for field in fields:
+            try:
+                rows = (db.session.query(field)
+                        .filter(field.isnot(None))
+                        .distinct()
+                        .limit(_VOCABULARY_LIMIT)
+                        .all())
+            except Exception:
+                # A plugin table missing on a lean install must not break
+                # search. No vocabulary is worse than the alternative, not
+                # fatal.
+                logger.debug('vocabulary: skipped %s', field, exc_info=True)
+                continue
+            for (value,) in rows:
+                vocabulary.add(value)
+
+    cache.set('search_vocabulary', vocabulary, timeout=_VOCABULARY_TTL)
+    return vocabulary
+
+
+def _suggestion_for(query):
+    """A better spelling of `query`, or None. Never raises into a search."""
+    try:
+        return _search_vocabulary().suggest_query(query)
+    except Exception:
+        logger.warning('vocabulary lookup failed', exc_info=True)
+        return None
+
+
 @search_bp.route('', methods=['GET'])
 @jwt_required(optional=True)
 def global_search():
@@ -1106,8 +1171,13 @@ def global_search():
     # multi-word and cross-field matching changes may already have fixed a good
     # share of what used to fail. Guessing at that would be building for a
     # problem nobody has measured.
+    suggestion = None
     if not unique_results:
         logger.info('search-no-results query=%r', query)
+        # Offered, never applied. The response carries the better spelling and
+        # the caller decides: silently searching for something else is how
+        # somebody orders the wrong cartridge.
+        suggestion = _suggestion_for(query)
 
     # Limit total results
     unique_results = unique_results[:50]
@@ -1120,6 +1190,8 @@ def global_search():
         'total_all': total_all,
         'counts': type_counts,
     }
+    if suggestion:
+        response_data['suggestion'] = suggestion
 
     redirect = _check_smart_redirect(query, classification)
     if redirect:

tests/test_core/test_spellfix.py

@@ -107,3 +107,56 @@ def test_the_vocabulary_comes_from_the_data():
     assert fresh.suggest('renishw') is None
     fresh.add('Renishaw')
     assert fresh.suggest('renishw') == 'renishaw'
+
+
+# ------------------------------------------------------- wired into search
+
+def test_a_search_that_finds_nothing_offers_a_spelling(client, db, auth_headers):
+    """End to end: the vocabulary is harvested from real rows, and a misspelled
+    query that returns no results comes back with a better spelling."""
+    from shopdb.core.models import Vendor
+    from shopdb.extensions import cache
+
+    db.session.add(Vendor(vendor='Keyence'))
+    db.session.commit()
+    cache.delete('search_vocabulary')
+
+    resp = client.get('/api/search?q=keyecne', headers=auth_headers)
+    assert resp.status_code == 200
+    data = resp.get_json()['data']
+    assert data['results'] == []
+    assert data.get('suggestion') == 'keyence'
+
+
+def test_a_search_that_finds_something_offers_nothing(client, db, auth_headers):
+    from shopdb.core.models import Vendor
+    from shopdb.extensions import cache
+
+    db.session.add(Vendor(vendor='Keyence'))
+    db.session.commit()
+    cache.delete('search_vocabulary')
+
+    resp = client.get('/api/search?q=keyence', headers=auth_headers)
+    assert 'suggestion' not in resp.get_json()['data']
+
+
+def test_a_mistyped_asset_number_is_not_corrected_to_a_real_one(
+        client, db, auth_headers):
+    """The guard, through the endpoint. CSF17 exists; CSF18 does not; and the
+    answer to "CSF18" must be no results and NO suggestion, not a different
+    bay."""
+    from shopdb.core.models import Asset, AssetType
+    from shopdb.extensions import cache
+
+    assettype = AssetType.query.first() or AssetType(assettype='machine')
+    if not assettype.assettypeid:
+        db.session.add(assettype)
+        db.session.flush()
+    db.session.add(Asset(assetnumber='CSF17', assettypeid=assettype.assettypeid))
+    db.session.commit()
+    cache.delete('search_vocabulary')
+
+    resp = client.get('/api/search?q=CSF18', headers=auth_headers)
+    data = resp.get_json()['data']
+    assert data['results'] == []
+    assert 'suggestion' not in data