From 1593ee820421b7cbdf9cfec714fca1ff1eca55b0 Mon Sep 17 00:00:00 2001 From: cproudlock Date: Fri, 21 Aug 2026 10:41:14 -0400 Subject: [PATCH] Offer a spelling when a search finds nothing The corrector landed in fa0f6d7 unwired, pending a week of zero-result logs to say what it should be aimed at. Attaching it now instead: the guards that make it safe here are principled rather than tuned to any particular data, so the logs would confirm the shape of the problem without changing the answer. ONLY ON ZERO RESULTS. A search that finds something pays for none of this - no vocabulary build, no scoring, no extra query. The vocabulary is harvested from columns people actually search - asset numbers and names, vendors, models, applications, asset types - cached ten minutes and capped per column. Long enough that someone mistyping twice does not rebuild it, short enough that a vendor entered this morning is correctable before lunch. The cap is there because this exists to catch a typo against the names in use, not to mirror the database into memory to answer one query. SUGGESTED, NEVER APPLIED. The response carries the better spelling and the caller decides. Searching for something else on a person's behalf is how somebody orders the wrong cartridge. FAILS SOFT. A missing column on a lean install, or anything else, returns no suggestion and the search answers normally. No vocabulary is worse than the alternative; it is not fatal. Three tests through the endpoint rather than only the unit: a misspelling gets a suggestion, a search that succeeds gets none, and CSF18 against an existing CSF17 returns no results AND no suggestion - the digit guard holding across the whole stack, which is the one that would cost somebody a wrong bay. The zero-result logging stays. It answers "what can people not find", which the corrector does not: a typo is only one of the reasons a search comes back empty, and the others are vocabulary nobody has entered and records that do not exist. NOT DONE: the search page does not render the suggestion yet, so this is invisible to a person until that Vue change lands. --- shopdb/core/api/search.py | 74 +++++++++++++++++++++++++++++++- tests/test_core/test_spellfix.py | 53 +++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/shopdb/core/api/search.py b/shopdb/core/api/search.py index a6c63c1..90dd66e 100644 --- a/shopdb/core/api/search.py +++ b/shopdb/core/api/search.py @@ -9,13 +9,14 @@ from flask import Blueprint, request, current_app from flask_jwt_extended import jwt_required from sqlalchemy.orm import joinedload -from shopdb.extensions import db +from shopdb.extensions import db, cache from shopdb.core.models import ( Application, Setting, Asset, AssetType, Communication, Vendor, Model, CustomField, CustomFieldValue ) from shopdb.core.api.settings import get_cached_settings +from shopdb.core.services.spellfix import Vocabulary from shopdb.utils.responses import success_response logger = logging.getLogger(__name__) @@ -1003,6 +1004,70 @@ def _check_smart_redirect(query, classification): return None +# How long a harvested vocabulary is reused. Long enough that a burst of +# searching does not rebuild it, short enough that something entered this +# morning is correctable before lunch. +_VOCABULARY_TTL = 600 + +# Per-column ceiling on harvested terms. A correction is only offered when a +# search found NOTHING, so the vocabulary exists to catch a typo against the +# names people actually use - not to mirror the database. Without a bound this +# would load every asset number on a site into memory to answer one query. +_VOCABULARY_LIMIT = 5000 + + +def _search_vocabulary(): + """Words worth correcting a search against, harvested from the data. + + From the DATA and not a dictionary, because no dictionary holds Genspect, + Telesis, Keyence or wax-trace - and because a vendor added this morning is + then correctable this morning. + + Cached: this runs only on a search that found nothing, but a person who + mistypes once usually mistypes twice. + """ + cached = cache.get('search_vocabulary') + if cached is not None: + return cached + + vocabulary = Vocabulary() + columns = ( + (Asset, (Asset.assetnumber, Asset.name)), + (Vendor, (Vendor.vendor,)), + (Model, (Model.modelnumber,)), + (Application, (Application.appname,)), + (AssetType, (AssetType.assettype,)), + ) + for model, fields in columns: + for field in fields: + try: + rows = (db.session.query(field) + .filter(field.isnot(None)) + .distinct() + .limit(_VOCABULARY_LIMIT) + .all()) + except Exception: + # A plugin table missing on a lean install must not break + # search. No vocabulary is worse than the alternative, not + # fatal. + logger.debug('vocabulary: skipped %s', field, exc_info=True) + continue + for (value,) in rows: + vocabulary.add(value) + + cache.set('search_vocabulary', vocabulary, timeout=_VOCABULARY_TTL) + return vocabulary + + +def _suggestion_for(query): + """A better spelling of `query`, or None. Never raises into a search.""" + try: + return _search_vocabulary().suggest_query(query) + except Exception: + logger.warning('vocabulary lookup failed', exc_info=True) + return None + + @search_bp.route('', methods=['GET']) @jwt_required(optional=True) def global_search(): @@ -1106,8 +1171,13 @@ def global_search(): # multi-word and cross-field matching changes may already have fixed a good # share of what used to fail. Guessing at that would be building for a # problem nobody has measured. + suggestion = None if not unique_results: logger.info('search-no-results query=%r', query) + # Offered, never applied. The response carries the better spelling and + # the caller decides: silently searching for something else is how + # somebody orders the wrong cartridge. + suggestion = _suggestion_for(query) # Limit total results unique_results = unique_results[:50] @@ -1120,6 +1190,8 @@ def global_search(): 'total_all': total_all, 'counts': type_counts, } + if suggestion: + response_data['suggestion'] = suggestion redirect = _check_smart_redirect(query, classification) if redirect: diff --git a/tests/test_core/test_spellfix.py b/tests/test_core/test_spellfix.py index faea9b3..1f8b756 100644 --- a/tests/test_core/test_spellfix.py +++ b/tests/test_core/test_spellfix.py @@ -107,3 +107,56 @@ def test_the_vocabulary_comes_from_the_data(): assert fresh.suggest('renishw') is None fresh.add('Renishaw') assert fresh.suggest('renishw') == 'renishaw' + + +# ------------------------------------------------------- wired into search + +def test_a_search_that_finds_nothing_offers_a_spelling(client, db, auth_headers): + """End to end: the vocabulary is harvested from real rows, and a misspelled + query that returns no results comes back with a better spelling.""" + from shopdb.core.models import Vendor + from shopdb.extensions import cache + + db.session.add(Vendor(vendor='Keyence')) + db.session.commit() + cache.delete('search_vocabulary') + + resp = client.get('/api/search?q=keyecne', headers=auth_headers) + assert resp.status_code == 200 + data = resp.get_json()['data'] + assert data['results'] == [] + assert data.get('suggestion') == 'keyence' + + +def test_a_search_that_finds_something_offers_nothing(client, db, auth_headers): + from shopdb.core.models import Vendor + from shopdb.extensions import cache + + db.session.add(Vendor(vendor='Keyence')) + db.session.commit() + cache.delete('search_vocabulary') + + resp = client.get('/api/search?q=keyence', headers=auth_headers) + assert 'suggestion' not in resp.get_json()['data'] + + +def test_a_mistyped_asset_number_is_not_corrected_to_a_real_one( + client, db, auth_headers): + """The guard, through the endpoint. CSF17 exists; CSF18 does not; and the + answer to "CSF18" must be no results and NO suggestion, not a different + bay.""" + from shopdb.core.models import Asset, AssetType + from shopdb.extensions import cache + + assettype = AssetType.query.first() or AssetType(assettype='machine') + if not assettype.assettypeid: + db.session.add(assettype) + db.session.flush() + db.session.add(Asset(assetnumber='CSF17', assettypeid=assettype.assettypeid)) + db.session.commit() + cache.delete('search_vocabulary') + + resp = client.get('/api/search?q=CSF18', headers=auth_headers) + data = resp.get_json()['data'] + assert data['results'] == [] + assert 'suggestion' not in data