Guides image + draft gating, audit retention, ADR-015 gate, backup credential redaction

shopdb/core/api/auditlogs.py

@@ -4,7 +4,8 @@ from flask import Blueprint, request
 from flask_jwt_extended import jwt_required
 
 from shopdb.core.models import AuditLog
-from shopdb.utils.responses import success_response
+from shopdb.extensions import db
+from shopdb.utils.responses import success_response, error_response, ErrorCodes
 from shopdb.utils.authz import require_permission
 
 auditlogs_bp = Blueprint('auditlogs', __name__)
@@ -194,3 +195,74 @@ def db_func_count_by(column):
     ).group_by(column).all()
 
     return {r[0]: r[1] for r in results}
+
+
+@auditlogs_bp.route('/retention', methods=['GET'])
+@jwt_required()
+@require_permission('audit.view')
+def get_retention():
+    """Size of the log and what the configured retention would remove.
+
+    Drives the Site & Facility card: an admin should see the number of entries a
+    purge deletes BEFORE pressing anything.
+    """
+    from shopdb.core.services import auditretention
+    days = auditretention.configureddays()
+    return success_response(auditretention.stats(days or None))
+
+
+@auditlogs_bp.route('/purge', methods=['POST'])
+@jwt_required()
+@require_permission('audit.purge')
+def purge_auditlogs():
+    """Delete entries older than the retention window.
+
+    Separate permission from audit.view: reading the log and destroying it are
+    not the same act, and plenty of people need the first.
+
+    Batch-capped, because this runs inside a request that a proxy will time out.
+    An incomplete purge returns complete=false and the number still to go rather
+    than reporting a finished job - the caller re-posts, or runs
+    `flask audit purge --apply` on the server, which has no cap.
+    """
+    from flask import request as flask_request
+    from shopdb.core.services import auditretention
+
+    body = flask_request.get_json(silent=True) or {}
+    days = body.get('days')
+    if days is None:
+        days = auditretention.configureddays()
+    try:
+        days = int(days)
+    except (TypeError, ValueError):
+        days = 0
+    if days <= 0:
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'No retention period is configured. Set one in Site & Facility '
+            'settings before purging.')
+
+    dryrun = bool(body.get('dryrun'))
+    result = auditretention.purge(
+        days, dryrun=dryrun,
+        maxbatches=None if dryrun else auditretention.MAXBATCHES_REQUEST)
+
+    if dryrun:
+        message = '{:,} entry(s) would be deleted'.format(result['matched'])
+    elif result['complete']:
+        message = 'Deleted {:,} entry(s)'.format(result['deleted'])
+    else:
+        message = ('Deleted {:,} entry(s); {:,} still older than the cutoff. '
+                   'Run it again to continue.').format(
+                       result['deleted'], result['remaining'])
+
+    # The purge is itself an audited act, and this row postdates the cutoff so
+    # it survives the purge that created it.
+    if not dryrun and result['deleted']:
+        from shopdb.api import audit_log
+        audit_log(action='deleted', entitytype='AuditLog',
+                  entityname='purge older than {} days'.format(days),
+                  changes={'deleted': result['deleted'],
+                           'cutoff': result['cutoff']})
+
+    return success_response(result, message=message)

shopdb/core/api/search.py

@@ -61,6 +61,32 @@ def _word_match(query, *columns, extra=None):
     return db.and_(*[per_word(w) for w in words])
 
 
+def _notefailure(domain, exc):
+    """Record that one search domain failed this request, and log it.
+
+    Every domain below catches broadly and returns [] so one broken table cannot
+    take the whole search box down. That is right, but on its own it makes a
+    DOWN domain indistinguishable from an EMPTY one: the technician sees "no
+    results", is told they may have misspelled it, and concludes the record does
+    not exist. It does; the search just could not look.
+
+    The names collected here travel back in the response so the UI can say which
+    part of the search did not run.
+    """
+    from flask import g
+    logger.error(f"{domain} search failed: {exc}")
+    failures = getattr(g, '_searchfailures', None)
+    if failures is None:
+        failures = g._searchfailures = []
+    if domain not in failures:
+        failures.append(domain)
+
+
+def _searchfailures():
+    from flask import g
+    return list(getattr(g, '_searchfailures', []) or [])
+
+
 def _require_enabled(name):
     """Raise ImportError when the named plugin is disabled.
 
@@ -72,13 +98,12 @@ def _require_enabled(name):
     if pm and not pm.registry.is_enabled(name):
         raise ImportError(f'{name} plugin disabled')
 
-# Shipped GE defaults. Settings override these per-site; identical fallbacks
-# live here so this consumer works even if the settings seed has not run.
-SERVICENOW_URL_DEFAULT = (
-    'https://geaerospaceqa.service-now.com/now/nav/ui/search/'
-    '0f8b85d0c7922010099a308dc7c2606a/params/search-term/{ticket}/'
-    'global-search-data-config-id/c861cea2c7022010099a308dc7c26041/'
-)
+# Fallbacks for when the settings seed has not run. The ServiceNow URL is BLANK
+# on purpose and must stay that way: a hardcoded tenant here would link one
+# site's technicians into another site's ServiceNow, and the code below already
+# treats a blank URL as "ServiceNow not configured" and renders tickets as plain
+# text. There is no sensible cross-site default for a tenant URL.
+SERVICENOW_URL_DEFAULT = ''
 SERVICENOW_PREFIXES_DEFAULT = 'GEINC,GECHG,GERIT,GESCT'
 EMPLOYEEID_PATTERN_DEFAULT = r'^\d{9}$'
 
@@ -108,9 +133,12 @@ def _get_search_integrations():
         prefixes_raw = SERVICENOW_PREFIXES_DEFAULT
     prefixes = [p.strip() for p in str(prefixes_raw).split(',') if p.strip()]
 
+    # Missing means unconfigured, and unconfigured means OFF - matching the
+    # shipped default. Defaulting to True here made a site with no ServiceNow
+    # settings at all behave as though the integration were on.
     servicenow_enabled = settings.get('servicenow_enabled')
     if servicenow_enabled is None:
-        servicenow_enabled = True
+        servicenow_enabled = False
 
     servicenow_url = settings.get('servicenow_search_url')
     if servicenow_url is None:
@@ -251,7 +279,7 @@ def _search_applications(query, search_term):
                 'relevance': relevance
             })
     except Exception as e:
-        logger.error(f"Application search failed: {e}")
+        _notefailure("Application", e)
     return results
 
 
@@ -264,43 +292,53 @@ def _search_guides(query, search_term):
     is a LIKE against it: crude, but it finds the guide, and a guide corpus is
     tens of documents rather than millions of rows.
 
-    PUBLISHED ONLY. A draft is deliberately invisible to readers, and search is
-    the one place that rule would otherwise leak - somebody would find a
-    half-written procedure through the search box and follow it.
+    PUBLISHED ONLY, and published TEXT only. A draft is deliberately invisible
+    to readers, and search is the one place that rule would otherwise leak -
+    somebody would find a half-written procedure through the search box and
+    follow it.
+
+    Restricting to published GUIDES is not enough on its own, which is how this
+    first shipped: title/summary/tags/blocksjson on the Guide row are the
+    WORKING copy. A published guide with unpublished edits would match on a word
+    that exists only in the draft, and hand the reader a hit that is not in the
+    page when they open it. So the match, the title and the summary all come
+    from the published revision.
     """
     results = []
     try:
         _require_enabled('guides')
-        from plugins.guides.models import Guide
+        from plugins.guides.models import Guide, GuideRevision
 
-        rows = Guide.query.filter(
+        published = db.aliased(GuideRevision)
+        rows = db.session.query(Guide, published).join(
+            published, Guide.publishedrevisionid == published.guiderevisionid
+        ).filter(
             Guide.isactive == True,
-            Guide.publishedrevisionid.isnot(None),
-            _word_match(query, Guide.title, Guide.summary, Guide.tags,
-                        Guide.blocksjson)
+            _word_match(query, published.title, published.summary,
+                        published.tags, published.blocksjson)
         ).order_by(Guide.modifieddate.desc()).limit(20).all()
 
-        for guide in rows:
+        for guide, revision in rows:
             # A title match outranks a body match: someone searching "marker"
             # wants the guide called that, not every guide mentioning one.
             relevance = 10
-            if query.lower() in (guide.title or '').lower():
+            if query.lower() in (revision.title or '').lower():
                 relevance += 20
-            if guide.tags and query.lower() in guide.tags.lower():
+            if revision.tags and query.lower() in revision.tags.lower():
                 relevance += 15
 
             results.append({
                 'type': 'guide',
                 'id': guide.guideid,
-                'title': guide.title,
-                'subtitle': guide.summary,
+                'title': revision.title,
+                'subtitle': revision.summary,
                 'url': f"/guides/{guide.slug}",
                 'relevance': relevance
             })
     except ImportError:
         pass  # guides plugin absent or disabled
     except Exception as e:
-        logger.error(f"Guides search failed: {e}")
+        _notefailure("Guides", e)
     return results
 
 
@@ -363,7 +401,7 @@ def _search_knowledgebase(query, search_term):
     except ImportError:
         pass  # knowledgebase plugin absent or disabled
     except Exception as e:
-        logger.error(f"KnowledgeBase search failed: {e}")
+        _notefailure("KnowledgeBase", e)
     return results
 
 
@@ -433,7 +471,7 @@ def _search_employees(query, search_term):
                 'relevance': relevance
             })
     except Exception as e:
-        logger.error(f"Employee search failed: {e}")
+        _notefailure("Employee", e)
     return results
 
 
@@ -483,7 +521,7 @@ def _search_assets(query, search_term):
 
             results.append(_get_asset_result(asset, query, relevance))
     except Exception as e:
-        logger.error(f"Asset search failed: {e}")
+        _notefailure("Asset", e)
     return results
 
 
@@ -523,7 +561,7 @@ def _search_measuringtools(query, search_term):
     except ImportError:
         pass  # measuringtools plugin absent or disabled
     except Exception as e:
-        logger.error(f"Measuring tool search failed: {e}")
+        _notefailure("Measuring tool", e)
     return results
 
 
@@ -574,7 +612,7 @@ def _search_usbdevices(query, search_term):
     except ImportError:
         pass  # usb plugin absent or disabled
     except Exception as e:
-        logger.error(f"USB device search failed: {e}")
+        _notefailure("USB device", e)
     return results
 
 
@@ -623,7 +661,7 @@ def _search_printeditems(query, search_term):
     except ImportError:
         pass  # printedparts plugin absent or disabled
     except Exception as e:
-        logger.error(f"Printed item search failed: {e}")
+        _notefailure("Printed item", e)
     return results
 
 
@@ -658,7 +696,7 @@ def _search_customfields(query, search_term):
             result['subtitle'] = field.label
             results.append(result)
     except Exception as e:
-        logger.error(f"Custom field search failed: {e}")
+        _notefailure("Custom field", e)
     return results
 
 
@@ -686,7 +724,7 @@ def _search_by_ip(query, search_term):
             result['subtitle'] = comm.ipaddress
             results.append(result)
     except Exception as e:
-        logger.error(f"IP search failed: {e}")
+        _notefailure("IP", e)
     return results
 
 
@@ -718,7 +756,7 @@ def _search_subnets(query):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Subnet search failed: {e}")
+        _notefailure("Subnet", e)
     return results
 
 
@@ -746,7 +784,7 @@ def _search_hostnames(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Computer hostname search failed: {e}")
+        _notefailure("Computer hostname", e)
 
     # Search Printers
     try:
@@ -770,7 +808,7 @@ def _search_hostnames(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Printer hostname search failed: {e}")
+        _notefailure("Printer hostname", e)
 
     # Search Network Devices
     try:
@@ -793,7 +831,7 @@ def _search_hostnames(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Network device hostname search failed: {e}")
+        _notefailure("Network device hostname", e)
 
     return results
 
@@ -841,7 +879,7 @@ def _search_notifications(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Notification search failed: {e}")
+        _notefailure("Notification", e)
     return results
 
 
@@ -875,7 +913,7 @@ def _search_vendor_model_type(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Machine vendor/model/type search failed: {e}")
+        _notefailure("Machine vendor/model/type", e)
 
     # Printers: vendor, model, printertype
     try:
@@ -903,7 +941,7 @@ def _search_vendor_model_type(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Printer vendor/model/type search failed: {e}")
+        _notefailure("Printer vendor/model/type", e)
 
     # Network Devices: vendor, networkdevicetype
     try:
@@ -929,7 +967,7 @@ def _search_vendor_model_type(query, search_term):
     except ImportError:
         pass
     except Exception as e:
-        logger.error(f"Network device vendor/type search failed: {e}")
+        _notefailure("Network device vendor/type", e)
 
     return results
 
@@ -1250,12 +1288,24 @@ def global_search():
     # multi-word and cross-field matching changes may already have fixed a good
     # share of what used to fail. Guessing at that would be building for a
     # problem nobody has measured.
+    degraded = _searchfailures()
+
     suggestion = None
     if not unique_results:
         logger.info('search-no-results query=%r', query)
         # Offered, never applied. The response carries the better spelling and
         # the caller decides: silently searching for something else is how
         # somebody orders the wrong cartridge.
+        #
+        # Still offered on a degraded run, because the response also carries
+        # `degraded` and the UI states it ABOVE the suggestion - the reader is
+        # told the search was incomplete before being offered a spelling.
+        #
+        # Suppressing it instead was tried and is worse: a domain that is
+        # chronically unreachable (an external HR directory, say) would silently
+        # disable spelling help for every search, for ever, and nothing would
+        # say why. Naming the failure is the honest fix; withholding the
+        # suggestion just removes a second useful signal.
         suggestion = _suggestion_for(query)
 
     # Limit total results
@@ -1271,6 +1321,10 @@ def global_search():
     }
     if suggestion:
         response_data['suggestion'] = suggestion
+    if degraded:
+        # The UI says which part of the search did not run. Without this a
+        # broken domain reads as "that record does not exist".
+        response_data['degraded'] = degraded
 
     redirect = _check_smart_redirect(query, classification)
     if redirect:

shopdb/core/api/settings.py

@@ -522,6 +522,13 @@ def build_default_settings():
             'category': 'site',
             'description': 'Public base URL of this site INCLUDING any subpath it is served under, e.g. https://shopdb.example.net or https://host/shopdb. Consumers append the in-app route to this and add nothing themselves, so a subpath left off here produces links that 404. Blank = use the browsing origin.'
         },
+        {
+            'key': 'audit_retentiondays',
+            'value': '0',
+            'valuetype': 'integer',
+            'category': 'site',
+            'description': 'Delete audit log entries older than this many days. 0 (default) keeps everything forever. The audit log is append-only and grows for as long as the site runs, so a long-lived site eventually needs a limit here; nothing is deleted until one is set, and the settings page shows how many entries a purge would remove before it runs.'
+        },
         {
             'key': 'facility_name',
             'value': '',
@@ -727,20 +734,22 @@ def build_default_settings():
     defaults = sitedefaults + brandingdefaults + printingdefaults + identifierdefaults + searchdefaults + mapdefaults + [
         # ServiceNow ticket links. Each instance points these at its own
         # ServiceNow tenant; blank/disabled renders tickets as plain text.
+        #
+        # Shipped OFF and BLANK, like the Zabbix block below. These defaults
+        # used to be one tenant's QA instance, which is wrong twice over: a
+        # sister site adopting this product would silently link its technicians
+        # at somebody else's ServiceNow, and even at the originating site a QA
+        # tenant is not where a real ticket lives.
         {
             'key': 'servicenow_enabled',
-            'value': 'true',
+            'value': 'false',
             'valuetype': 'boolean',
             'category': 'integrations',
             'description': 'Enable ServiceNow ticket recognition and links in search and dashboards'
         },
         {
             'key': 'servicenow_search_url',
-            'value': (
-                'https://geaerospaceqa.service-now.com/now/nav/ui/search/'
-                '0f8b85d0c7922010099a308dc7c2606a/params/search-term/{ticket}/'
-                'global-search-data-config-id/c861cea2c7022010099a308dc7c26041/'
-            ),
+            'value': '',
             'valuetype': 'string',
             'category': 'integrations',
             'description': 'ServiceNow global-search URL template. {ticket} is the ticket number.'
@@ -754,14 +763,14 @@ def build_default_settings():
         },
         {
             'key': 'servicenow_incident_url',
-            'value': 'https://geaerospaceqa.service-now.com/now/nav/ui/search/0f8b85d0c7922010099a308dc7c2606a/params/search-term/{ticket}/global-search-data-config-id/c861cea2c7022010099a308dc7c26041/',
+            'value': '',
             'valuetype': 'string',
             'category': 'integrations',
             'description': 'ServiceNow incident URL template. {ticket} is the incident number.'
         },
         {
             'key': 'servicenow_change_url',
-            'value': 'https://geaerospaceqa.service-now.com/now/nav/ui/search/0f8b85d0c7922010099a308dc7c2606a/params/search-term/{ticket}/global-search-data-config-id/c861cea2c7022010099a308dc7c26041/',
+            'value': '',
             'valuetype': 'string',
             'category': 'integrations',
             'description': 'ServiceNow change-request URL template. {ticket} is the change number.'