Ship the equipment catalog so a new site does not start empty

`flask seed reference-data` wrote a dozen generic model types and no vendors or
models at all, so adopting this platform began by retyping a catalog another
site had already spent a year building. That is the largest single obstacle to
standing a new facility up.

scripts/export_catalog.py dumps the catalog from a live instance to
shopdb/data/catalog.json, and `flask seed catalog` loads it. What travels:

  vendors 53, models 128, modelsupplies 146, modeltypes 35, machinetypes 21,
  computertypes 10, printertypes 9, networkdevicetypes 5, locationtypes 11,
  operatingsystems 14, measuringtooltypes 8, notificationtypes 3,
  accessprotocols 3

The 146 printer supplies are the most useful part after the models themselves:
every toner, drum and maintenance kit with its part number, colour, capacity
tier and page yield, already matched to the right model, instead of somebody
reading them off spent cartridges.

IDEMPOTENT and ADDITIVE. Records match on a natural key - a vendor by name, a
model by vendor plus model number, a supply by model plus part number - so a
second run adds nothing, and it never updates or deletes: a site that corrected
a description or pointed a model at its own photo keeps its version.

Catalog only. No assets, locations, employees, business units or anything with a
serial number: nobody wants one plant's machines appearing at another. Vendor
contact details are excluded too, since a rep's name and number belong to
whoever holds that relationship. supportteams, printerdrivers and customfields
are site-specific and deliberately absent.

Models and supplies reference their vendor by NAME rather than id, because ids
differ between databases and an id-keyed catalog would silently attach part
numbers to the wrong printer.

The installer offers it as a tick-box on a new "Starter data" page, defaulting
to on, passing -SeedCatalog to stage 3. Offered rather than assumed: a site that
machines nothing does not want 21 machine types cluttering its dropdowns.

Verified by loading into an empty database and running twice: every group
populated on the first pass, "Catalog already present, nothing to add" on the
second.

deploy/windows/installer/ShopDBFlask.iss

@@ -201,6 +201,7 @@ var
   // the check takes a few seconds, runs hidden, and gives no sign of life.
   CheckingPage:   TOutputProgressWizardPage;
   PluginPage:     TInputOptionWizardPage;
+  CatalogPage:    TInputOptionWizardPage;
   PluginNames:    TArrayOfString;
   PluginPageReady: Boolean;
   DbPageReady:     Boolean;
@@ -397,7 +398,21 @@ begin
     if Trim(PluginNames[I]) <> '' then
       PluginPage.Add(PluginLabel(Trim(PluginNames[I])));
 
-  DbChoicePage := CreateInputOptionPage(PluginPage.ID,
+  // Offered, not forced. A site that machines nothing inherits 21 machine types
+  // it will never use, and a catalog nobody wanted is just clutter in every
+  // dropdown. Ticked by default because the alternative is retyping a vendor and
+  // model list another site already built.
+  CatalogPage := CreateInputOptionPage(PluginPage.ID,
+    'Starter data', 'Load the shared equipment catalog?',
+    'Vendors, model numbers and the type lists, as built up across the other ' +
+    'sites. It only ADDS - nothing existing is changed or removed - and it can ' +
+    'be loaded later with "flask seed catalog". No assets, locations or people ' +
+    'are included; this is catalogue information only.',
+    False, False);
+  CatalogPage.Add('Load the shared vendor and model catalog');
+  CatalogPage.Values[0] := True;
+
+  DbChoicePage := CreateInputOptionPage(CatalogPage.ID,
     'Database', 'Where should ShopDB-Flask store its data?',
     'Most sites already run MySQL for the existing shopdb application. If so, ' +
     'choose the second option - installing a second server would collide on ' +
@@ -1177,6 +1192,8 @@ begin
             ' -OnFailure never' +
             ' -ClientIpSource ' + ClientIpSourceArg +
             ' -SitePlugins "' + SelectedPlugins + '"';
+  if CatalogPage.Values[0] then
+    Common := Common + ' -SeedCatalog';
 
   // Subpath deployment: an IIS Application under the existing site instead of a
   // site of its own. Appended here because Pascal has no conditional expression.

deploy/windows/installer/shopdb-install.ps1

@@ -127,6 +127,10 @@ param(
     # client reads as 127.0.0.1: the GE-Enforce IP allowlist, the dashboard
     # visitor-location lookup and per-host login rate limiting all break quietly.
     [ValidateSet('direct','proxy')] [string] $ClientIpSource = 'direct',
+    # Load the shared vendor/model catalog during stage 3. Offered by the
+    # wizard rather than assumed: a site that machines nothing does not want
+    # 21 machine types cluttering its dropdowns.
+    [switch] $SeedCatalog,
     # Required before this installer will alter an installation it did not
     # create. Everything here is built for a greenfield server: it makes its own
     # Python, its own venv and its own IIS objects, and its upgrade path assumes
@@ -1747,6 +1751,14 @@ build to get a matching pair, then send the install log to support.
             Invoke-Native $Flask @('seed',$seed) "seed $seed"
         }
 
+        # Additive and idempotent, so a re-run adds only what is genuinely new
+        # and never overwrites a description somebody corrected here.
+        if ($SeedCatalog) {
+            Write-Log 'flask seed catalog   (shared vendor and model catalog)'
+            $catalogOut = Invoke-Native $Flask @('seed','catalog') 'seed catalog' -OkExit 0,1
+            $catalogOut | Select-Object -Last 12 | ForEach-Object { Write-Log "    $_" }
+        }
+
         # Plugin registry starts empty on a fresh box. apply-profile resolves the
         # declared set's hard-dependency closure and installs + enables it in
         # dependency order, idempotently (ADR-013). It fails loudly if the profile

scripts/export_catalog.py

@@ -0,0 +1,173 @@
+"""Export the reference CATALOG from a live instance to a portable JSON file.
+
+A new site starts with almost nothing: `flask seed reference-data` writes a
+dozen generic model types and no vendors or models at all. So the first thing
+anyone does at a new facility is retype a vendor and model list that another
+facility already spent a year building. This lets that catalog travel.
+
+CATALOG ONLY. Vendors, models and the type vocabularies are descriptions of
+equipment that exists in the world, and are the same at every site. Assets,
+locations, employees, business units and anything with a serial number are that
+site's own records and are deliberately NOT exported - nobody wants West
+Jefferson's machines appearing at another plant.
+
+Vendor contact details are also skipped: a rep's name and phone number belong to
+whoever has the relationship, not to the catalog.
+
+    python scripts/export_catalog.py                     # to shopdb/data/catalog.json
+    python scripts/export_catalog.py --out mine.json
+    python scripts/export_catalog.py --pretty            # readable diffs in git
+
+Load it on another instance with:  flask seed catalog
+"""
+
+import argparse
+import json
+import os
+import sys
+from collections import OrderedDict
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+DEFAULT_OUT = os.path.join('shopdb', 'data', 'catalog.json')
+
+
+def export(connection):
+    from sqlalchemy import inspect, text
+
+    inspector = inspect(connection)
+    tables = set(inspector.get_table_names())
+    data = OrderedDict()
+
+    def rows(sql):
+        return [dict(r._mapping) for r in connection.execute(text(sql))]
+
+    def has(table):
+        return table in tables
+
+    # --- type vocabularies -------------------------------------------------
+    # Each is a small controlled list, and a site with none of them cannot even
+    # classify what it owns.
+    if has('modeltypes'):
+        data['modeltypes'] = rows("""
+            SELECT modeltype, category, description, icon
+            FROM modeltypes ORDER BY modeltype""")
+    for table, column in (('machinetypes', 'machinetype'),
+                          ('computertypes', 'computertype'),
+                          ('printertypes', 'printertype'),
+                          ('networkdevicetypes', 'networkdevicetype'),
+                          ('locationtypes', 'locationtype')):
+        if has(table):
+            extra = ', color' if 'color' in {c['name'] for c in inspector.get_columns(table)} else ''
+            data[table] = rows('SELECT %s, description%s FROM %s ORDER BY %s'
+                               % (column, extra, table, column))
+
+    if has('operatingsystems'):
+        columns = {c['name'] for c in inspector.get_columns('operatingsystems')}
+        wanted = [c for c in ('osname', 'osversion', 'osfamily', 'description') if c in columns]
+        data['operatingsystems'] = rows('SELECT %s FROM operatingsystems ORDER BY osname'
+                                        % ', '.join(wanted))
+
+    # --- vendors -----------------------------------------------------------
+    # Name and website only. Contact person, phone and address are a site's own
+    # relationship with that vendor, not a property of the vendor.
+    if has('vendors'):
+        columns = {c['name'] for c in inspector.get_columns('vendors')}
+        wanted = [c for c in ('vendor', 'website', 'description') if c in columns]
+        data['vendors'] = rows('SELECT %s FROM vendors ORDER BY vendor' % ', '.join(wanted))
+
+    # --- models ------------------------------------------------------------
+    # Referenced by NAME, not id: ids differ between databases, and a catalog
+    # keyed on them would silently attach models to the wrong vendor.
+    if has('models'):
+        data['models'] = rows("""
+            SELECT m.modelnumber          AS modelnumber,
+                   v.vendor               AS vendor,
+                   mt.modeltype           AS modeltype,
+                   m.description          AS description,
+                   m.documentationurl     AS documentationurl,
+                   m.imageurl             AS imageurl
+            FROM models m
+            LEFT JOIN vendors v     ON m.vendorid = v.vendorid
+            LEFT JOIN modeltypes mt ON m.modeltypeid = mt.modeltypeid
+            ORDER BY v.vendor, m.modelnumber""")
+
+    # --- printer supplies --------------------------------------------------
+    # The single most useful thing to inherit after the models themselves: a new
+    # site with printers gets every toner, drum and maintenance-kit part number
+    # already matched to the right model, instead of somebody reading them off
+    # cartridges. Keyed by the model's natural key for the same reason models
+    # are keyed by vendor name.
+    if has('modelsupplies') and has('models'):
+        data['modelsupplies'] = rows("""
+            SELECT mo.modelnumber   AS modelnumber,
+                   v.vendor         AS vendor,
+                   s.supplytype     AS supplytype,
+                   s.color          AS color,
+                   s.capacitytier   AS capacitytier,
+                   s.partnumber     AS partnumber,
+                   s.marketingname  AS marketingname,
+                   s.pageyield      AS pageyield,
+                   s.notes          AS notes
+            FROM modelsupplies s
+            JOIN models mo      ON s.modelnumberid = mo.modelnumberid
+            LEFT JOIN vendors v ON mo.vendorid = v.vendorid
+            ORDER BY v.vendor, mo.modelnumber, s.partnumber""")
+
+    # --- small generic vocabularies ---------------------------------------
+    # Each is a controlled list that means the same thing at any site.
+    if has('measuringtooltypes'):
+        data['measuringtooltypes'] = rows(
+            'SELECT name, description, color FROM measuringtooltypes ORDER BY name')
+    if has('notificationtypes'):
+        data['notificationtypes'] = rows(
+            'SELECT typename, typedescription, typecolor FROM notificationtypes ORDER BY typename')
+    if has('accessprotocols'):
+        columns = {c['name'] for c in inspector.get_columns('accessprotocols')}
+        wanted = [c for c in ('name', 'scheme', 'defaultport', 'linktemplate') if c in columns]
+        data['accessprotocols'] = rows('SELECT %s FROM accessprotocols ORDER BY name'
+                                       % ', '.join(wanted))
+
+    # Drop empty optional values so the file stays readable and a NULL does not
+    # arrive as the string "None".
+    for key, records in data.items():
+        cleaned = []
+        for record in records:
+            cleaned.append({k: v for k, v in record.items() if v not in (None, '')})
+        data[key] = cleaned
+    return data
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--out', default=DEFAULT_OUT, help='output file')
+    parser.add_argument('--pretty', action='store_true',
+                        help='indent the JSON so git diffs are readable')
+    args = parser.parse_args()
+
+    from shopdb import create_app
+    from shopdb.extensions import db
+
+    app = create_app()
+    with app.app_context():
+        data = export(db.session.connection())
+
+    directory = os.path.dirname(os.path.abspath(args.out))
+    if directory:
+        os.makedirs(directory, exist_ok=True)
+    with open(args.out, 'w', encoding='utf-8') as handle:
+        json.dump(data, handle, indent=2 if args.pretty else None,
+                  ensure_ascii=False, sort_keys=False)
+        handle.write('\n')
+
+    print('Wrote %s' % args.out)
+    for key, records in data.items():
+        print('  %-22s %d' % (key, len(records)))
+    print('')
+    print('Load it on another instance with:  flask seed catalog')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())