shopdb-flask/scripts/site_imports/wjf/run.py

"""WJ classic-ASP -> flask reference import loader (site-specific).

Run against a THROWAWAY import database:

    DATABASE_URL='mysql+pymysql://root:PW@127.0.0.1:3306/shopdb_flask_import?charset=utf8mb4' \\
        venv/bin/python -m scripts.site_imports.wjf.run --stages reference,employees

Stages are idempotent (re-running resolves existing rows by natural key), so a
crashed run resumes. See scripts/site_imports/wjf/harness.py for the contract
this drives, and scratchpad/IMPORT-PLAN.md for the full mapping + decisions.

Implemented: reference (vendors, businessunits, operatingsystems), employees.
TODO stages (assets hub + dependents + network + usb) are stubbed - they need
the machineid->assetid crosswalk this harness persists.
"""

import argparse

from .harness import Harness


def _upsert(h, path, payload, unique_field, list_path=None):
    """POST payload; on 409 resolve the existing row by its unique field.
    Returns the new/existing id (best-effort key 'id' variants), or None."""
    status, data = h.post(path, payload)
    if status in (200, 201):
        return _id_of(data)
    if status == 409:
        _, rows = h.get(list_path or path)
        items = rows if isinstance(rows, list) else rows.get('items', rows)
        for row in (items or []):
            if str(row.get(unique_field, '')).strip().lower() == \
                    str(payload[unique_field]).strip().lower():
                return _id_of(row)
    return None


def _id_of(row):
    for key in ('vendorid', 'businessunitid', 'osid', 'id'):
        if isinstance(row, dict) and row.get(key) is not None:
            return row[key]
    return None


def stage_reference(h):
    """Seed the reference tier that later FKs resolve against, capturing
    legacy-id -> new-id crosswalks."""
    counts = {}

    vendors = h.source.rows('shopdb_src',
                            'SELECT vendorid, vendor FROM vendors WHERE isactive=1')
    for v in vendors:
        newid = _upsert(h, '/api/vendors', {'vendor': v['vendor']}, 'vendor')
        if newid:
            h.ids.put('vendor', v['vendorid'], newid)
    counts['vendors'] = h.ids.count('vendor')

    bus = h.source.rows('shopdb_src',
                        'SELECT businessunitid, businessunit FROM businessunits WHERE isactive=1')
    for b in bus:
        newid = _upsert(h, '/api/businessunits', {'businessunit': b['businessunit']},
                        'businessunit')
        if newid:
            h.ids.put('businessunit', b['businessunitid'], newid)
    counts['businessunits'] = h.ids.count('businessunit')

    oses = h.source.rows('shopdb_src',
                        'SELECT osid, operatingsystem FROM operatingsystems')
    for o in oses:
        name = (o['operatingsystem'] or '').strip()
        if not name:
            continue
        newid = _upsert(h, '/api/operatingsystems', {'osname': name}, 'osname')
        if newid:
            h.ids.put('os', o['osid'], newid)
    counts['operatingsystems'] = h.ids.count('os')

    return counts


def stage_employees(h):
    """Bulk-upsert the employee directory (selfhosted). Photos are deferred
    (per the import decisions); occurrences are out of scope."""
    from shopdb.core.models import Setting
    Setting.set('employee_directory_mode', 'selfhosted', valuetype='string',
                category='employees')
    from shopdb.extensions import db
    db.session.commit()

    rows = h.source.rows('wjf_employees_src',
                        'SELECT SSO, First_Name, Last_Name, Team, Role FROM employees')
    lines = ['SSO,First_Name,Last_Name,Team,Role']
    for r in rows:
        vals = [str(r['SSO']), (r['First_Name'] or '').strip(),
                (r['Last_Name'] or '').strip(), (r['Team'] or '').strip(),
                (r['Role'] or '').strip()]
        lines.append(','.join(v.replace(',', ' ') for v in vals))
    status, data = h.post('/api/employees/directory/import', {'csv': '\n'.join(lines)})
    return {'employees_posted': len(rows), 'result': data}


STAGES = {
    'reference': stage_reference,
    'employees': stage_employees,
    # TODO: 'models', 'applications', 'assets', 'dependents', 'network', 'usb'
}


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--stages', default='reference,employees',
                        help='comma list of stages to run')
    args = parser.parse_args()

    h = Harness()
    print(f'== WJ import loader (target: {h.app.config["SQLALCHEMY_DATABASE_URI"].split("/")[-1]}) ==')
    try:
        for name in args.stages.split(','):
            name = name.strip()
            if name not in STAGES:
                print(f'  skip unknown stage: {name}')
                continue
            result = STAGES[name](h)
            print(f'  [{name}] {result}')
        if h.errors:
            print(f'\n  {len(h.errors)} endpoint error(s):')
            for path, code, _payload, data in h.errors[:10]:
                print(f'    {code} {path}: {data}')
    finally:
        h.close()


if __name__ == '__main__':
    main()