Merge remote-tracking branch 'origin/main' into wilm

shopdb/core/api/assets.py

@@ -506,6 +506,17 @@ def update_asset(asset_id: int):
     if not data:
         return error_response(ErrorCodes.VALIDATION_ERROR, 'No data provided')
 
+    # An UPDATE may not blank the asset number. Create validates it and the
+    # column is NOT NULL, but the conflict check below only runs when the value
+    # DIFFERS, and '' never collides with anything - so a payload carrying an
+    # empty assetnumber wrote it straight through. A form that loaded blank
+    # (a failed reference load, a partial fetch) then saved the blank over a
+    # real record.
+    if 'assetnumber' in data and not (data['assetnumber'] or '').strip():
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'assetnumber cannot be empty')
+
     # Check for conflicting assetnumber
     if 'assetnumber' in data and data['assetnumber'] != asset.assetnumber:
         if Asset.query.filter_by(assetnumber=data['assetnumber']).first():

shopdb/core/api/search.py

@@ -9,13 +9,14 @@ from flask import Blueprint, request, current_app
 from flask_jwt_extended import jwt_required
 from sqlalchemy.orm import joinedload
 
-from shopdb.extensions import db
+from shopdb.extensions import db, cache
 from shopdb.core.models import (
     Application, Setting,
     Asset, AssetType, Communication, Vendor, Model,
     CustomField, CustomFieldValue
 )
 from shopdb.core.api.settings import get_cached_settings
+from shopdb.core.services.spellfix import Vocabulary
 from shopdb.utils.responses import success_response
 
 logger = logging.getLogger(__name__)
@@ -31,7 +32,7 @@ search_bp = Blueprint('search', __name__)
 # cannot reorder. Relevance is applied afterwards in Python, over a stable set.
 
 
-def _word_match(query, *columns):
+def _word_match(query, *columns, extra=None):
     """SQL clause matching rows that contain EVERY word of the query, each word
     in any of the columns, in any order.
 
@@ -39,14 +40,25 @@ def _word_match(query, *columns):
     multi-word search returned nothing. This splits the query into words and ANDs
     them (OR across the columns per word): 'CSF Roles' matches a row with 'CSF'
     and 'Roles' anywhere in the searched fields.
+
+    `extra` is an optional callable taking one word and returning a further
+    clause to OR in for that word. Use it where a word may be satisfied by a
+    RELATED row rather than a column of this table - a knowledge base article
+    matching on its topic's name, say. Splitting per word matters there: without
+    it a word can only ever be matched by the columns, so a query naming the
+    topic and a keyword ('CMM Community') matches neither side alone.
     """
     words = [w for w in query.split() if w]
     if not words:
         words = ['']
-    return db.and_(*[
-        db.or_(*[c.ilike(f'%{w}%') for c in columns])
-        for w in words
-    ])
+
+    def per_word(word):
+        clauses = [c.ilike(f'%{word}%') for c in columns]
+        if extra is not None:
+            clauses.append(extra(word))
+        return db.or_(*clauses)
+
+    return db.and_(*[per_word(w) for w in words])
 
 
 def _require_enabled(name):
@@ -253,6 +265,10 @@ def _search_knowledgebase(query, search_term):
     subject, which reads as though it were still in service.
 
     A null topic still matches. Not every article is about an application.
+
+    The topic's NAME is searched too, per word. An article tagged 'community'
+    under the CMM topic has neither word in both of its own columns, so a search
+    for 'CMM Community' found it in neither the plugin's listing nor here.
     """
     results = []
     try:
@@ -260,12 +276,24 @@ def _search_knowledgebase(query, search_term):
         from plugins.knowledgebase.models import KnowledgeBase
         retired = db.session.query(Application.appid).filter(
             Application.isactive.is_(False))
+
+        def topic_named(word):
+            """Articles whose topic is an ACTIVE application named like word.
+
+            Active only, for the same reason the retired filter above exists: a
+            decommissioned application is not a topic anyone should be offered.
+            """
+            return KnowledgeBase.appid.in_(
+                db.session.query(Application.appid).filter(
+                    Application.appname.ilike(f'%{word}%'),
+                    Application.isactive.is_(True)))
+
         kb_articles = KnowledgeBase.query.filter(
             KnowledgeBase.isactive == True,
             db.or_(KnowledgeBase.appid.is_(None),
                    KnowledgeBase.appid.notin_(retired)),
             _word_match(query, KnowledgeBase.shortdescription,
-                        KnowledgeBase.keywords)
+                        KnowledgeBase.keywords, extra=topic_named)
         ).order_by(KnowledgeBase.clicks.desc(),
                    KnowledgeBase.linkid).limit(20).all()
 
@@ -976,6 +1004,94 @@ def _check_smart_redirect(query, classification):
     return None
 
 
+# How long a harvested vocabulary is reused. Long enough that a burst of
+# searching does not rebuild it, short enough that something entered this
+# morning is correctable before lunch.
+_VOCABULARY_TTL = 600
+
+# Per-column ceiling on harvested terms. A correction is only offered when a
+# search found NOTHING, so the vocabulary exists to catch a typo against the
+# names people actually use - not to mirror the database. Without a bound this
+# would load every asset number on a site into memory to answer one query.
+_VOCABULARY_LIMIT = 5000
+
+
+def _search_vocabulary():
+    """Words worth correcting a search against, harvested from the data.
+
+    From the DATA and not a dictionary, because no dictionary holds Genspect,
+    Telesis, Keyence or wax-trace - and because a vendor added this morning is
+    then correctable this morning.
+
+    Cached: this runs only on a search that found nothing, but a person who
+    mistypes once usually mistypes twice.
+    """
+    cached = cache.get('search_vocabulary')
+    if cached is not None:
+        return cached
+
+    vocabulary = Vocabulary()
+    columns = (
+        (Asset, (Asset.assetnumber, Asset.name)),
+        (Vendor, (Vendor.vendor,)),
+        (Model, (Model.modelnumber,)),
+        (Application, (Application.appname,)),
+        (AssetType, (AssetType.assettype,)),
+    )
+    for model, fields in columns:
+        for field in fields:
+            try:
+                rows = (db.session.query(field)
+                        .filter(field.isnot(None))
+                        .distinct()
+                        .limit(_VOCABULARY_LIMIT)
+                        .all())
+            except Exception:
+                # A plugin table missing on a lean install must not break
+                # search. No vocabulary is worse than the alternative, not
+                # fatal.
+                logger.debug('vocabulary: skipped %s', field, exc_info=True)
+                continue
+            for (value,) in rows:
+                vocabulary.add(value)
+
+    cache.set('search_vocabulary', vocabulary, timeout=_VOCABULARY_TTL)
+    return vocabulary
+
+
+def _suggestion_for(query):
+    """A better spelling of `query`, or None. Never raises into a search."""
+    try:
+        return _search_vocabulary().suggest_query(query)
+    except Exception:
+        logger.warning('vocabulary lookup failed', exc_info=True)
+        return None
+
+
+@search_bp.route('/suggest', methods=['GET'])
+@jwt_required(optional=True)
+def search_suggest():
+    """A better spelling for a query that found nothing, or null.
+
+    ITS OWN ENDPOINT, deliberately, rather than a `suggestion` key added to
+    every list response. Thirty routes take a search parameter across a dozen
+    files, so folding it into each one is a large change today and a thing every
+    future plugin has to remember. A page that finds no rows calls this instead;
+    a page that has not adopted it simply shows no suggestion, which is what it
+    showed before.
+
+    Returns null rather than a poor guess. The caller offers "did you mean" and
+    a person decides - nothing searches on their behalf.
+    """
+    query = request.args.get('q', '').strip()
+    if not query or len(query) < 2 or len(query) > 200:
+        return success_response({'query': query, 'suggestion': None})
+    return success_response({
+        'query': query,
+        'suggestion': _suggestion_for(query),
+    })
+
+
 @search_bp.route('', methods=['GET'])
 @jwt_required(optional=True)
 def global_search():
@@ -1072,6 +1188,21 @@ def global_search():
 
     total_all = len(unique_results)
 
+    # A search that found NOTHING is the only evidence of what people cannot
+    # find, and it is not otherwise recorded anywhere. Logged at INFO with a
+    # stable prefix so a week of it can be grepped into a list, which is what
+    # should decide whether spelling correction is worth wiring in - the recent
+    # multi-word and cross-field matching changes may already have fixed a good
+    # share of what used to fail. Guessing at that would be building for a
+    # problem nobody has measured.
+    suggestion = None
+    if not unique_results:
+        logger.info('search-no-results query=%r', query)
+        # Offered, never applied. The response carries the better spelling and
+        # the caller decides: silently searching for something else is how
+        # somebody orders the wrong cartridge.
+        suggestion = _suggestion_for(query)
+
     # Limit total results
     unique_results = unique_results[:50]
 
@@ -1083,6 +1214,8 @@ def global_search():
         'total_all': total_all,
         'counts': type_counts,
     }
+    if suggestion:
+        response_data['suggestion'] = suggestion
 
     redirect = _check_smart_redirect(query, classification)
     if redirect: