shopdb-flask/scripts/export_catalog.py

"""Export the reference CATALOG from a live instance to a portable JSON file.

A new site starts with almost nothing: `flask seed reference-data` writes a
dozen generic model types and no vendors or models at all. So the first thing
anyone does at a new facility is retype a vendor and model list that another
facility already spent a year building. This lets that catalog travel.

CATALOG ONLY. Vendors, models and the type vocabularies are descriptions of
equipment that exists in the world, and are the same at every site. Assets,
locations, employees, business units and anything with a serial number are that
site's own records and are deliberately NOT exported - nobody wants West
Jefferson's machines appearing at another plant.

Vendor contact details are also skipped: a rep's name and phone number belong to
whoever has the relationship, not to the catalog.

    python scripts/export_catalog.py                     # to shopdb/data/catalog.json
    python scripts/export_catalog.py --out mine.json
    python scripts/export_catalog.py --pretty            # readable diffs in git

Load it on another instance with:  flask seed catalog
"""

import argparse
import json
import os
import sys
from pathlib import Path
from collections import OrderedDict

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from dotenv import load_dotenv

# Load .env before anything reads config. wsgi.py does this for the served app
# and the `flask` CLI does it for itself, but a plain `python scripts/...` run
# gets neither: DATABASE_URL stays unset, FLASK_ENV defaults to development,
# and config falls back to a hardcoded root@localhost URL. That fails on a real
# deployment with a bare 1045 Access denied for 'root', naming a user the site
# does not use - and on a box where those root credentials happen to work, it
# is worse, because the script silently reads or WRITES the wrong database.
load_dotenv(Path(__file__).resolve().parent.parent / '.env')


DEFAULT_OUT = os.path.join('shopdb', 'data', 'catalog.json')


def export(connection):
    from sqlalchemy import inspect, text

    inspector = inspect(connection)
    tables = set(inspector.get_table_names())
    data = OrderedDict()

    def rows(sql):
        return [dict(r._mapping) for r in connection.execute(text(sql))]

    def has(table):
        return table in tables

    # --- type vocabularies -------------------------------------------------
    # Each is a small controlled list, and a site with none of them cannot even
    # classify what it owns.
    if has('modeltypes'):
        data['modeltypes'] = rows("""
            SELECT modeltype, category, description, icon
            FROM modeltypes ORDER BY modeltype""")
    for table, column in (('machinetypes', 'machinetype'),
                          ('computertypes', 'computertype'),
                          ('printertypes', 'printertype'),
                          ('networkdevicetypes', 'networkdevicetype'),
                          ('locationtypes', 'locationtype')):
        if has(table):
            extra = ', color' if 'color' in {c['name'] for c in inspector.get_columns(table)} else ''
            data[table] = rows('SELECT %s, description%s FROM %s ORDER BY %s'
                               % (column, extra, table, column))

    if has('operatingsystems'):
        columns = {c['name'] for c in inspector.get_columns('operatingsystems')}
        wanted = [c for c in ('osname', 'osversion', 'osfamily', 'description') if c in columns]
        data['operatingsystems'] = rows('SELECT %s FROM operatingsystems ORDER BY osname'
                                        % ', '.join(wanted))

    # --- vendors -----------------------------------------------------------
    # Name and website only. Contact person, phone and address are a site's own
    # relationship with that vendor, not a property of the vendor.
    if has('vendors'):
        columns = {c['name'] for c in inspector.get_columns('vendors')}
        wanted = [c for c in ('vendor', 'website', 'description') if c in columns]
        data['vendors'] = rows('SELECT %s FROM vendors ORDER BY vendor' % ', '.join(wanted))

    # --- models ------------------------------------------------------------
    # Referenced by NAME, not id: ids differ between databases, and a catalog
    # keyed on them would silently attach models to the wrong vendor.
    if has('models'):
        data['models'] = rows("""
            SELECT m.modelnumber          AS modelnumber,
                   v.vendor               AS vendor,
                   mt.modeltype           AS modeltype,
                   m.description          AS description,
                   m.documentationurl     AS documentationurl,
                   m.imageurl             AS imageurl
            FROM models m
            LEFT JOIN vendors v     ON m.vendorid = v.vendorid
            LEFT JOIN modeltypes mt ON m.modeltypeid = mt.modeltypeid
            ORDER BY v.vendor, m.modelnumber""")

    # --- printer supplies --------------------------------------------------
    # The single most useful thing to inherit after the models themselves: a new
    # site with printers gets every toner, drum and maintenance-kit part number
    # already matched to the right model, instead of somebody reading them off
    # cartridges. Keyed by the model's natural key for the same reason models
    # are keyed by vendor name.
    if has('modelsupplies') and has('models'):
        data['modelsupplies'] = rows("""
            SELECT mo.modelnumber   AS modelnumber,
                   v.vendor         AS vendor,
                   s.supplytype     AS supplytype,
                   s.color          AS color,
                   s.capacitytier   AS capacitytier,
                   s.partnumber     AS partnumber,
                   s.marketingname  AS marketingname,
                   s.pageyield      AS pageyield,
                   s.notes          AS notes
            FROM modelsupplies s
            JOIN models mo      ON s.modelnumberid = mo.modelnumberid
            LEFT JOIN vendors v ON mo.vendorid = v.vendorid
            ORDER BY v.vendor, mo.modelnumber, s.partnumber""")

    # --- small generic vocabularies ---------------------------------------
    # Each is a controlled list that means the same thing at any site.
    if has('notificationtypes'):
        data['notificationtypes'] = rows(
            'SELECT typename, typedescription, typecolor FROM notificationtypes ORDER BY typename')
    if has('accessprotocols'):
        columns = {c['name'] for c in inspector.get_columns('accessprotocols')}
        wanted = [c for c in ('name', 'scheme', 'defaultport', 'linktemplate') if c in columns]
        data['accessprotocols'] = rows('SELECT %s FROM accessprotocols ORDER BY name'
                                       % ', '.join(wanted))

    # Drop empty optional values so the file stays readable and a NULL does not
    # arrive as the string "None".
    for key, records in data.items():
        cleaned = []
        for record in records:
            cleaned.append({k: v for k, v in record.items() if v not in (None, '')})
        data[key] = cleaned
    return data


def main():
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--out', default=DEFAULT_OUT, help='output file')
    parser.add_argument('--pretty', action='store_true',
                        help='indent the JSON so git diffs are readable')
    args = parser.parse_args()

    from shopdb import create_app
    from shopdb.extensions import db

    app = create_app()
    with app.app_context():
        data = export(db.session.connection())

    directory = os.path.dirname(os.path.abspath(args.out))
    if directory:
        os.makedirs(directory, exist_ok=True)
    with open(args.out, 'w', encoding='utf-8') as handle:
        json.dump(data, handle, indent=2 if args.pretty else None,
                  ensure_ascii=False, sort_keys=False)
        handle.write('\n')

    print('Wrote %s' % args.out)
    for key, records in data.items():
        print('  %-22s %d' % (key, len(records)))
    print('')
    print('Load it on another instance with:  flask seed catalog')
    return 0


if __name__ == '__main__':
    sys.exit(main())