Catalog seeding, blank-unique fix, slides rotation, console repair

shopdb/__init__.py

@@ -88,6 +88,12 @@ def create_app(config_name: str = None) -> Flask:
     # Initialize extensions
     init_extensions(app)
 
+    # An optional unique column accepts any number of NULLs but exactly one
+    # empty string, so a second blank code collided and became a 500. Normalise
+    # blank to NULL once, at the mapper, rather than in each endpoint.
+    from .utils.blankunique import register_blank_unique_normaliser
+    register_blank_unique_normaliser()
+
     # Initialize plugin manager
     with app.app_context():
         plugin_manager.init_app(app, db)
@@ -186,6 +192,31 @@ def register_error_handlers(app: Flask):
     from .utils.responses import error_response, ErrorCodes
     from .exceptions import ShopDBException
 
+    # A uniqueness violation is the caller's problem, not a server fault. Without
+    # this it surfaced as a bare 500 with a SQLAlchemy traceback in the log and
+    # nothing usable on screen - the operator saw "internal server error" for
+    # having reused a code that was already taken.
+    from sqlalchemy.exc import IntegrityError
+
+    @app.errorhandler(IntegrityError)
+    def handle_integrity_error(error):
+        from .extensions import db
+        db.session.rollback()
+        message = str(getattr(error, 'orig', error))
+        app.logger.warning('integrity error: %s', message)
+        if 'Duplicate entry' in message or 'UNIQUE constraint' in message:
+            return error_response(
+                ErrorCodes.CONFLICT,
+                'That value is already in use. Codes and identifiers must be unique.',
+                http_code=409)
+        if 'foreign key constraint' in message.lower():
+            return error_response(
+                ErrorCodes.VALIDATION_ERROR,
+                'That record refers to something that does not exist, or is still in use elsewhere.',
+                http_code=400)
+        return error_response(ErrorCodes.VALIDATION_ERROR,
+                              'The database rejected that change.', http_code=400)
+
     @app.errorhandler(ShopDBException)
     def handle_shopdb_exception(error):
         http_codes = {

shopdb/cli/__init__.py

@@ -10,6 +10,58 @@ def db_cli():
     pass
 
 
+@db_cli.command('seed-state')
+@with_appcontext
+def seed_state():
+    """Report whether the seed data a working install depends on is present.
+
+    A server whose seeds never ran does not fail politely: settings endpoints
+    answer 404 for keys that were never created and most pages answer 500, which
+    reads as a broken application rather than an unfinished install. This gives
+    the operator console something definite to test, so it can say "run repair"
+    instead of leaving somebody to infer it from unrelated errors.
+
+    Prints one line per group and exits non-zero if anything is missing, so it
+    can be used as a gate as well as read by a person.
+    """
+    import sys
+    from shopdb.extensions import db
+    from sqlalchemy import text
+
+    # Sentinels, not exhaustive counts. Each is created by one of the three seed
+    # commands, so a zero here means that command never ran.
+    checks = [
+        ('permissions',    'SELECT COUNT(*) FROM permissions',   'flask seed permissions'),
+        ('settings',       'SELECT COUNT(*) FROM settings',      'flask seed settings'),
+        ('asset types',    'SELECT COUNT(*) FROM assettypes',    'flask seed reference-data'),
+        ('location types', 'SELECT COUNT(*) FROM locationtypes', 'flask seed reference-data'),
+    ]
+
+    missing = []
+    for label, sql, remedy in checks:
+        try:
+            count = db.session.execute(text(sql)).scalar() or 0
+        except Exception as exc:
+            click.echo(click.style('  MISSING ', fg='red') +
+                       '%s - table unreadable (%s)' % (label, type(exc).__name__))
+            missing.append((label, remedy))
+            continue
+        if count == 0:
+            click.echo(click.style('  MISSING ', fg='red') +
+                       '%s - none present, run: %s' % (label, remedy))
+            missing.append((label, remedy))
+        else:
+            click.echo(click.style('  OK      ', fg='green') + '%s (%d)' % (label, count))
+
+    if missing:
+        click.echo('')
+        click.echo(click.style('%d group(s) missing. This server is not fully provisioned.'
+                               % len(missing), fg='red'))
+        sys.exit(1)
+    click.echo('')
+    click.echo(click.style('Seed data present.', fg='green'))
+
+
 @db_cli.command('create-all')
 @with_appcontext
 def create_all():
@@ -147,6 +199,189 @@ def seed_cli():
     pass
 
 
+@seed_cli.command('catalog')
+@click.option('--file', 'path', default=None,
+              help='catalog JSON to load (default shopdb/data/catalog.json)')
+@click.option('--dry-run', is_flag=True, help='report what would be added, write nothing')
+@with_appcontext
+def seed_catalog(path, dry_run):
+    """Load the shared vendor/model catalog shipped with the product.
+
+    `seed reference-data` writes a dozen generic model types and no vendors or
+    models, so every new site began by retyping a catalog another site had
+    already built. This loads that catalog instead.
+
+    IDEMPOTENT and ADDITIVE. Records are matched by natural key - a vendor by
+    name, a model by vendor plus model number, a type by its name - so running
+    it twice adds nothing the second time. It never updates or deletes an
+    existing record: a site that has corrected a description or pointed a model
+    at its own photo keeps its version.
+
+    Catalog only. Nothing here identifies a site: no assets, locations,
+    employees or serial numbers.
+    """
+    import json
+    import os
+    from shopdb.extensions import db
+    from shopdb.core.models import ModelType, OperatingSystem, LocationType, Vendor, Model
+
+    if not path:
+        path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                            'data', 'catalog.json')
+    if not os.path.isfile(path):
+        click.echo(click.style('No catalog file at %s' % path, fg='red'))
+        raise SystemExit(1)
+
+    with open(path, encoding='utf-8') as handle:
+        data = json.load(handle)
+
+    added = {}
+
+    def note(kind, count):
+        if count:
+            added[kind] = added.get(kind, 0) + count
+
+    def simple(key, model, column):
+        """Type vocabularies: one row per name."""
+        records = data.get(key) or []
+        count = 0
+        for record in records:
+            name = record.get(column)
+            if not name:
+                continue
+            if db.session.query(model).filter_by(**{column: name}).first():
+                continue
+            db.session.add(model(**{k: v for k, v in record.items()
+                                    if hasattr(model, k)}))
+            count += 1
+        note(key, count)
+
+    simple('modeltypes', ModelType, 'modeltype')
+    simple('locationtypes', LocationType, 'locationtype')
+    simple('operatingsystems', OperatingSystem, 'osname')
+
+    # Per-plugin type tables exist only where that plugin is installed, so they
+    # are loaded through the ORM registry rather than imported directly - a lean
+    # build without printers must not fail here.
+    plugin_types = [
+        ('machinetypes', 'machinetype'),
+        ('computertypes', 'computertype'),
+        ('printertypes', 'printertype'),
+        ('networkdevicetypes', 'networkdevicetype'),
+    ]
+    by_table = {m.class_.__tablename__: m.class_ for m in db.Model.registry.mappers}
+    for key, column in plugin_types:
+        model = by_table.get(key)
+        if model is None:
+            continue
+        simple(key, model, column)
+
+    # Vendors before models, since a model resolves its vendor by name.
+    count = 0
+    for record in data.get('vendors') or []:
+        name = record.get('vendor')
+        if not name or Vendor.query.filter_by(vendor=name).first():
+            continue
+        db.session.add(Vendor(**{k: v for k, v in record.items() if hasattr(Vendor, k)}))
+        count += 1
+    note('vendors', count)
+    db.session.flush()
+
+    vendor_ids = {v.vendor: v.vendorid for v in Vendor.query.all()}
+    modeltype_ids = {m.modeltype: m.modeltypeid for m in ModelType.query.all()}
+
+    count = 0
+    skipped_vendor = 0
+    for record in data.get('models') or []:
+        modelnumber = record.get('modelnumber')
+        if not modelnumber:
+            continue
+        vendorid = vendor_ids.get(record.get('vendor'))
+        # The catalog's unique key is model number PLUS vendor, so the same
+        # number from two makers stays two records.
+        if Model.query.filter_by(modelnumber=modelnumber, vendorid=vendorid).first():
+            continue
+        if record.get('vendor') and vendorid is None:
+            skipped_vendor += 1
+            continue
+        db.session.add(Model(
+            modelnumber=modelnumber,
+            vendorid=vendorid,
+            modeltypeid=modeltype_ids.get(record.get('modeltype')),
+            description=record.get('description'),
+            documentationurl=record.get('documentationurl'),
+            imageurl=record.get('imageurl'),
+        ))
+        count += 1
+    note('models', count)
+
+    # Small plugin vocabularies, loaded through the registry so a lean build
+    # missing that plugin skips them instead of failing to import.
+    for key, model_table, column in (('measuringtooltypes', 'measuringtooltypes', 'name'),
+                                     ('notificationtypes', 'notificationtypes', 'typename'),
+                                     ('accessprotocols', 'accessprotocols', 'name')):
+        model = by_table.get(model_table)
+        if model is not None:
+            simple(key, model, column)
+
+    # Printer supplies. Resolved against the models loaded above, by the model's
+    # natural key - part numbers are useless attached to the wrong printer.
+    supply_model = by_table.get('modelsupplies')
+    if supply_model is not None and (data.get('modelsupplies') or []):
+        db.session.flush()
+        model_key = {}
+        for m in Model.query.all():
+            model_key[(m.modelnumber, m.vendorid)] = m.modelnumberid
+        vendor_ids = {v.vendor: v.vendorid for v in Vendor.query.all()}
+
+        count = 0
+        orphaned = 0
+        for record in data['modelsupplies']:
+            partnumber = record.get('partnumber')
+            modelnumber = record.get('modelnumber')
+            if not partnumber or not modelnumber:
+                continue
+            modelnumberid = model_key.get((modelnumber, vendor_ids.get(record.get('vendor'))))
+            if modelnumberid is None:
+                orphaned += 1
+                continue
+            if db.session.query(supply_model).filter_by(
+                    modelnumberid=modelnumberid, partnumber=partnumber).first():
+                continue
+            db.session.add(supply_model(
+                modelnumberid=modelnumberid,
+                supplytype=record.get('supplytype') or 'toner',
+                color=record.get('color') or 'none',
+                capacitytier=record.get('capacitytier') or 'standard',
+                partnumber=partnumber,
+                marketingname=record.get('marketingname'),
+                pageyield=record.get('pageyield'),
+                notes=record.get('notes'),
+            ))
+            count += 1
+        note('modelsupplies', count)
+        if orphaned:
+            click.echo(click.style('  %d supply record(s) skipped: their model is not in this catalog'
+                                   % orphaned, fg='yellow'))
+
+    if dry_run:
+        db.session.rollback()
+        click.echo(click.style('DRY RUN - nothing written.', fg='yellow'))
+    else:
+        db.session.commit()
+
+    if not added:
+        click.echo(click.style('Catalog already present, nothing to add.', fg='green'))
+    else:
+        for kind in sorted(added):
+            click.echo('  %-22s +%d' % (kind, added[kind]))
+        click.echo('')
+        click.echo(click.style('Catalog loaded from %s' % os.path.basename(path), fg='green'))
+    if skipped_vendor:
+        click.echo(click.style('  %d model(s) skipped: their vendor is not in this catalog'
+                               % skipped_vendor, fg='yellow'))
+
+
 @seed_cli.command('reference-data')
 @with_appcontext
 def seed_reference_data():

shopdb/utils/blankunique.py

@@ -0,0 +1,81 @@
+"""Store an empty optional unique field as NULL, not as an empty string.
+
+A column that is `unique=True` and nullable accepts any number of NULLs, which
+is what makes "optional but unique" work at all. It accepts exactly ONE empty
+string. So the first record saved with a blank code succeeds and the second
+fails with "Duplicate entry '' for key", surfacing as a 500 from an endpoint
+that did nothing wrong.
+
+That is not a business-unit problem. A dozen columns across core and the plugins
+are unique and nullable - asset numbers, host names, item codes, subnet names -
+and every one of them is a blank web form away from the same collision. Fixing
+them one endpoint at a time would leave the next one to be discovered by a user.
+
+So it is fixed once, at the mapper: before any insert or update, a string column
+that is unique and nullable and has been set to '' or whitespace is stored as
+NULL instead. An empty string carries no information in a unique column - there
+is no case where two records both meaning "blank" should collide - so nothing is
+lost by normalising it.
+
+Registered from create_app. Applies to every model, including plugin models
+loaded later, because it hooks the shared Mapper class rather than a list of
+models known at import time.
+"""
+
+from sqlalchemy import event, String, inspect
+from sqlalchemy.orm import Session
+
+_CACHE = {}
+
+
+def _blankable_columns(mapper):
+    """Unique, nullable, string columns on this mapper - worked out once each."""
+    cached = _CACHE.get(mapper)
+    if cached is not None:
+        return cached
+
+    columns = []
+    for prop in mapper.column_attrs:
+        for column in prop.columns:
+            if not column.unique or not column.nullable:
+                continue
+            # Only text. A unique nullable integer cannot be handed '' by a form
+            # without failing type coercion long before it reaches here.
+            if not isinstance(column.type, String):
+                continue
+            columns.append(prop.key)
+            break
+    _CACHE[mapper] = columns
+    return columns
+
+
+def _normalise_instance(obj):
+    try:
+        mapper = inspect(obj).mapper
+    except Exception:
+        return
+    for key in _blankable_columns(mapper):
+        value = getattr(obj, key, None)
+        if isinstance(value, str) and not value.strip():
+            setattr(obj, key, None)
+
+
+def _before_flush(session, _flush_context, _instances):
+    # Both new and modified: editing a record to clear its code has to become
+    # NULL for the same reason creating one with a blank code does.
+    for obj in list(session.new) + list(session.dirty):
+        _normalise_instance(obj)
+
+
+def register_blank_unique_normaliser():
+    """Hook every session flush, for every model including plugins'.
+
+    Listening on Session rather than on individual mappers means models imported
+    later - which is every plugin model - are covered without registration, and
+    it avoids depending on mapper-level event semantics that differ across
+    SQLAlchemy versions.
+    """
+    if getattr(register_blank_unique_normaliser, '_installed', False):
+        return
+    event.listen(Session, 'before_flush', _before_flush)
+    register_blank_unique_normaliser._installed = True