"""WJ classic-ASP -> flask reference import loader (site-specific). Run against a THROWAWAY import database: DATABASE_URL='mysql+pymysql://root:PW@127.0.0.1:3306/shopdb_flask_import?charset=utf8mb4' \\ venv/bin/python -m scripts.site_imports.wjf.run --stages reference,employees Stages are idempotent (re-running resolves existing rows by natural key), so a crashed run resumes. See scripts/site_imports/wjf/harness.py for the contract this drives, and scratchpad/IMPORT-PLAN.md for the full mapping + decisions. Implemented: reference (vendors, businessunits, operatingsystems), employees. TODO stages (assets hub + dependents + network + usb) are stubbed - they need the machineid->assetid crosswalk this harness persists. """ import argparse from .harness import Harness def _upsert(h, path, payload, unique_field, list_path=None): """POST payload; on 409 resolve the existing row by its unique field. Returns the new/existing id (best-effort key 'id' variants), or None.""" status, data = h.post(path, payload) if status in (200, 201): return _id_of(data) if status == 409: _, rows = h.get(list_path or path) items = rows if isinstance(rows, list) else rows.get('items', rows) for row in (items or []): if str(row.get(unique_field, '')).strip().lower() == \ str(payload[unique_field]).strip().lower(): return _id_of(row) return None def _id_of(row): for key in ('vendorid', 'businessunitid', 'osid', 'id'): if isinstance(row, dict) and row.get(key) is not None: return row[key] return None def stage_reference(h): """Seed the reference tier that later FKs resolve against, capturing legacy-id -> new-id crosswalks.""" counts = {} vendors = h.source.rows('shopdb_src', 'SELECT vendorid, vendor FROM vendors WHERE isactive=1') for v in vendors: newid = _upsert(h, '/api/vendors', {'vendor': v['vendor']}, 'vendor') if newid: h.ids.put('vendor', v['vendorid'], newid) counts['vendors'] = h.ids.count('vendor') bus = h.source.rows('shopdb_src', 'SELECT businessunitid, businessunit FROM businessunits WHERE isactive=1') for b in bus: newid = _upsert(h, '/api/businessunits', {'businessunit': b['businessunit']}, 'businessunit') if newid: h.ids.put('businessunit', b['businessunitid'], newid) counts['businessunits'] = h.ids.count('businessunit') oses = h.source.rows('shopdb_src', 'SELECT osid, operatingsystem FROM operatingsystems') for o in oses: name = (o['operatingsystem'] or '').strip() if not name: continue newid = _upsert(h, '/api/operatingsystems', {'osname': name}, 'osname') if newid: h.ids.put('os', o['osid'], newid) counts['operatingsystems'] = h.ids.count('os') return counts def stage_employees(h): """Bulk-upsert the employee directory (selfhosted). Photos are deferred (per the import decisions); occurrences are out of scope.""" from shopdb.core.models import Setting Setting.set('employee_directory_mode', 'selfhosted', valuetype='string', category='employees') from shopdb.extensions import db db.session.commit() rows = h.source.rows('wjf_employees_src', 'SELECT SSO, First_Name, Last_Name, Team, Role FROM employees') lines = ['SSO,First_Name,Last_Name,Team,Role'] for r in rows: vals = [str(r['SSO']), (r['First_Name'] or '').strip(), (r['Last_Name'] or '').strip(), (r['Team'] or '').strip(), (r['Role'] or '').strip()] lines.append(','.join(v.replace(',', ' ') for v in vals)) status, data = h.post('/api/employees/directory/import', {'csv': '\n'.join(lines)}) return {'employees_posted': len(rows), 'result': data} STAGES = { 'reference': stage_reference, 'employees': stage_employees, # TODO: 'models', 'applications', 'assets', 'dependents', 'network', 'usb' } def main(): parser = argparse.ArgumentParser() parser.add_argument('--stages', default='reference,employees', help='comma list of stages to run') args = parser.parse_args() h = Harness() print(f'== WJ import loader (target: {h.app.config["SQLALCHEMY_DATABASE_URI"].split("/")[-1]}) ==') try: for name in args.stages.split(','): name = name.strip() if name not in STAGES: print(f' skip unknown stage: {name}') continue result = STAGES[name](h) print(f' [{name}] {result}') if h.errors: print(f'\n {len(h.errors)} endpoint error(s):') for path, code, _payload, data in h.errors[:10]: print(f' {code} {path}: {data}') finally: h.close() if __name__ == '__main__': main()