diff --git a/shopdb/core/api/search.py b/shopdb/core/api/search.py index a6c63c1..90dd66e 100644 --- a/shopdb/core/api/search.py +++ b/shopdb/core/api/search.py @@ -9,13 +9,14 @@ from flask import Blueprint, request, current_app from flask_jwt_extended import jwt_required from sqlalchemy.orm import joinedload -from shopdb.extensions import db +from shopdb.extensions import db, cache from shopdb.core.models import ( Application, Setting, Asset, AssetType, Communication, Vendor, Model, CustomField, CustomFieldValue ) from shopdb.core.api.settings import get_cached_settings +from shopdb.core.services.spellfix import Vocabulary from shopdb.utils.responses import success_response logger = logging.getLogger(__name__) @@ -1003,6 +1004,70 @@ def _check_smart_redirect(query, classification): return None +# How long a harvested vocabulary is reused. Long enough that a burst of +# searching does not rebuild it, short enough that something entered this +# morning is correctable before lunch. +_VOCABULARY_TTL = 600 + +# Per-column ceiling on harvested terms. A correction is only offered when a +# search found NOTHING, so the vocabulary exists to catch a typo against the +# names people actually use - not to mirror the database. Without a bound this +# would load every asset number on a site into memory to answer one query. +_VOCABULARY_LIMIT = 5000 + + +def _search_vocabulary(): + """Words worth correcting a search against, harvested from the data. + + From the DATA and not a dictionary, because no dictionary holds Genspect, + Telesis, Keyence or wax-trace - and because a vendor added this morning is + then correctable this morning. + + Cached: this runs only on a search that found nothing, but a person who + mistypes once usually mistypes twice. + """ + cached = cache.get('search_vocabulary') + if cached is not None: + return cached + + vocabulary = Vocabulary() + columns = ( + (Asset, (Asset.assetnumber, Asset.name)), + (Vendor, (Vendor.vendor,)), + (Model, (Model.modelnumber,)), + (Application, (Application.appname,)), + (AssetType, (AssetType.assettype,)), + ) + for model, fields in columns: + for field in fields: + try: + rows = (db.session.query(field) + .filter(field.isnot(None)) + .distinct() + .limit(_VOCABULARY_LIMIT) + .all()) + except Exception: + # A plugin table missing on a lean install must not break + # search. No vocabulary is worse than the alternative, not + # fatal. + logger.debug('vocabulary: skipped %s', field, exc_info=True) + continue + for (value,) in rows: + vocabulary.add(value) + + cache.set('search_vocabulary', vocabulary, timeout=_VOCABULARY_TTL) + return vocabulary + + +def _suggestion_for(query): + """A better spelling of `query`, or None. Never raises into a search.""" + try: + return _search_vocabulary().suggest_query(query) + except Exception: + logger.warning('vocabulary lookup failed', exc_info=True) + return None + + @search_bp.route('', methods=['GET']) @jwt_required(optional=True) def global_search(): @@ -1106,8 +1171,13 @@ def global_search(): # multi-word and cross-field matching changes may already have fixed a good # share of what used to fail. Guessing at that would be building for a # problem nobody has measured. + suggestion = None if not unique_results: logger.info('search-no-results query=%r', query) + # Offered, never applied. The response carries the better spelling and + # the caller decides: silently searching for something else is how + # somebody orders the wrong cartridge. + suggestion = _suggestion_for(query) # Limit total results unique_results = unique_results[:50] @@ -1120,6 +1190,8 @@ def global_search(): 'total_all': total_all, 'counts': type_counts, } + if suggestion: + response_data['suggestion'] = suggestion redirect = _check_smart_redirect(query, classification) if redirect: diff --git a/tests/test_core/test_spellfix.py b/tests/test_core/test_spellfix.py index faea9b3..1f8b756 100644 --- a/tests/test_core/test_spellfix.py +++ b/tests/test_core/test_spellfix.py @@ -107,3 +107,56 @@ def test_the_vocabulary_comes_from_the_data(): assert fresh.suggest('renishw') is None fresh.add('Renishaw') assert fresh.suggest('renishw') == 'renishaw' + + +# ------------------------------------------------------- wired into search + +def test_a_search_that_finds_nothing_offers_a_spelling(client, db, auth_headers): + """End to end: the vocabulary is harvested from real rows, and a misspelled + query that returns no results comes back with a better spelling.""" + from shopdb.core.models import Vendor + from shopdb.extensions import cache + + db.session.add(Vendor(vendor='Keyence')) + db.session.commit() + cache.delete('search_vocabulary') + + resp = client.get('/api/search?q=keyecne', headers=auth_headers) + assert resp.status_code == 200 + data = resp.get_json()['data'] + assert data['results'] == [] + assert data.get('suggestion') == 'keyence' + + +def test_a_search_that_finds_something_offers_nothing(client, db, auth_headers): + from shopdb.core.models import Vendor + from shopdb.extensions import cache + + db.session.add(Vendor(vendor='Keyence')) + db.session.commit() + cache.delete('search_vocabulary') + + resp = client.get('/api/search?q=keyence', headers=auth_headers) + assert 'suggestion' not in resp.get_json()['data'] + + +def test_a_mistyped_asset_number_is_not_corrected_to_a_real_one( + client, db, auth_headers): + """The guard, through the endpoint. CSF17 exists; CSF18 does not; and the + answer to "CSF18" must be no results and NO suggestion, not a different + bay.""" + from shopdb.core.models import Asset, AssetType + from shopdb.extensions import cache + + assettype = AssetType.query.first() or AssetType(assettype='machine') + if not assettype.assettypeid: + db.session.add(assettype) + db.session.flush() + db.session.add(Asset(assetnumber='CSF17', assettypeid=assettype.assettypeid)) + db.session.commit() + cache.delete('search_vocabulary') + + resp = client.get('/api/search?q=CSF18', headers=auth_headers) + data = resp.get_json()['data'] + assert data['results'] == [] + assert 'suggestion' not in data