Search spelling suggestions across every list

shopdb/core/api/search.py

@@ -9,13 +9,14 @@ from flask import Blueprint, request, current_app
 from flask_jwt_extended import jwt_required
 from sqlalchemy.orm import joinedload
 
-from shopdb.extensions import db
+from shopdb.extensions import db, cache
 from shopdb.core.models import (
     Application, Setting,
     Asset, AssetType, Communication, Vendor, Model,
     CustomField, CustomFieldValue
 )
 from shopdb.core.api.settings import get_cached_settings
+from shopdb.core.services.spellfix import Vocabulary
 from shopdb.utils.responses import success_response
 
 logger = logging.getLogger(__name__)
@@ -1003,6 +1004,94 @@ def _check_smart_redirect(query, classification):
     return None
 
 
+# How long a harvested vocabulary is reused. Long enough that a burst of
+# searching does not rebuild it, short enough that something entered this
+# morning is correctable before lunch.
+_VOCABULARY_TTL = 600
+
+# Per-column ceiling on harvested terms. A correction is only offered when a
+# search found NOTHING, so the vocabulary exists to catch a typo against the
+# names people actually use - not to mirror the database. Without a bound this
+# would load every asset number on a site into memory to answer one query.
+_VOCABULARY_LIMIT = 5000
+
+
+def _search_vocabulary():
+    """Words worth correcting a search against, harvested from the data.
+
+    From the DATA and not a dictionary, because no dictionary holds Genspect,
+    Telesis, Keyence or wax-trace - and because a vendor added this morning is
+    then correctable this morning.
+
+    Cached: this runs only on a search that found nothing, but a person who
+    mistypes once usually mistypes twice.
+    """
+    cached = cache.get('search_vocabulary')
+    if cached is not None:
+        return cached
+
+    vocabulary = Vocabulary()
+    columns = (
+        (Asset, (Asset.assetnumber, Asset.name)),
+        (Vendor, (Vendor.vendor,)),
+        (Model, (Model.modelnumber,)),
+        (Application, (Application.appname,)),
+        (AssetType, (AssetType.assettype,)),
+    )
+    for model, fields in columns:
+        for field in fields:
+            try:
+                rows = (db.session.query(field)
+                        .filter(field.isnot(None))
+                        .distinct()
+                        .limit(_VOCABULARY_LIMIT)
+                        .all())
+            except Exception:
+                # A plugin table missing on a lean install must not break
+                # search. No vocabulary is worse than the alternative, not
+                # fatal.
+                logger.debug('vocabulary: skipped %s', field, exc_info=True)
+                continue
+            for (value,) in rows:
+                vocabulary.add(value)
+
+    cache.set('search_vocabulary', vocabulary, timeout=_VOCABULARY_TTL)
+    return vocabulary
+
+
+def _suggestion_for(query):
+    """A better spelling of `query`, or None. Never raises into a search."""
+    try:
+        return _search_vocabulary().suggest_query(query)
+    except Exception:
+        logger.warning('vocabulary lookup failed', exc_info=True)
+        return None
+
+
+@search_bp.route('/suggest', methods=['GET'])
+@jwt_required(optional=True)
+def search_suggest():
+    """A better spelling for a query that found nothing, or null.
+
+    ITS OWN ENDPOINT, deliberately, rather than a `suggestion` key added to
+    every list response. Thirty routes take a search parameter across a dozen
+    files, so folding it into each one is a large change today and a thing every
+    future plugin has to remember. A page that finds no rows calls this instead;
+    a page that has not adopted it simply shows no suggestion, which is what it
+    showed before.
+
+    Returns null rather than a poor guess. The caller offers "did you mean" and
+    a person decides - nothing searches on their behalf.
+    """
+    query = request.args.get('q', '').strip()
+    if not query or len(query) < 2 or len(query) > 200:
+        return success_response({'query': query, 'suggestion': None})
+    return success_response({
+        'query': query,
+        'suggestion': _suggestion_for(query),
+    })
+
+
 @search_bp.route('', methods=['GET'])
 @jwt_required(optional=True)
 def global_search():
@@ -1099,6 +1188,21 @@ def global_search():
 
     total_all = len(unique_results)
 
+    # A search that found NOTHING is the only evidence of what people cannot
+    # find, and it is not otherwise recorded anywhere. Logged at INFO with a
+    # stable prefix so a week of it can be grepped into a list, which is what
+    # should decide whether spelling correction is worth wiring in - the recent
+    # multi-word and cross-field matching changes may already have fixed a good
+    # share of what used to fail. Guessing at that would be building for a
+    # problem nobody has measured.
+    suggestion = None
+    if not unique_results:
+        logger.info('search-no-results query=%r', query)
+        # Offered, never applied. The response carries the better spelling and
+        # the caller decides: silently searching for something else is how
+        # somebody orders the wrong cartridge.
+        suggestion = _suggestion_for(query)
+
     # Limit total results
     unique_results = unique_results[:50]
 
@@ -1110,6 +1214,8 @@ def global_search():
         'total_all': total_all,
         'counts': type_counts,
     }
+    if suggestion:
+        response_data['suggestion'] = suggestion
 
     redirect = _check_smart_redirect(query, classification)
     if redirect: