diff --git a/scripts/site_imports/wjf/run.py b/scripts/site_imports/wjf/run.py index 3d81008..4d0c2a5 100644 --- a/scripts/site_imports/wjf/run.py +++ b/scripts/site_imports/wjf/run.py @@ -495,24 +495,187 @@ def stage_knowledgebase(h): return {'kb_articles': made} +def stage_locations(h): + """The 24 islocationonly rows -> core Locations. Crosswalk machineid -> + locationid (relationships can point at them).""" + made = 0 + for m in h.source.rows('shopdb_src', + 'SELECT machineid, machinenumber, alias FROM machines ' + 'WHERE machinetypeid=1 AND islocationonly=1'): + name = (m['alias'] or m['machinenumber'] or '').strip() + if not name: + continue + newid = _upsert(h, '/api/locations', {'locationname': name}, + 'locationname', 'locationid', '/api/locations') + if newid: + h.ids.put('location', m['machineid'], newid) + made += 1 + return {'locations': made} + + +def stage_relationships(h): + """Active machinerelationships -> asset relationships via the crosswalk. + Edges whose endpoints did not become assets (locations/skipped/dups) drop. + Dedups on the (source, target, type) triple.""" + # Seed relationship types (case-insensitive collation folds Controls onto + # the seeded 'controls', etc), capturing the crosswalk. + for t in h.source.rows('shopdb_src', + 'SELECT relationshiptypeid, relationshiptype FROM relationshiptypes WHERE isactive=1'): + name = (t['relationshiptype'] or '').strip() + if not name: + continue + newid = _upsert(h, '/api/assets/relationshiptypes', {'relationshiptype': name}, + 'relationshiptype', 'relationshiptypeid', '/api/assets/relationshiptypes') + if newid: + h.ids.put('relationshiptype', t['relationshiptypeid'], newid) + + seen = set() + made = dropped = 0 + for r in h.source.rows('shopdb_src', + 'SELECT machineid, related_machineid, relationshiptypeid ' + 'FROM machinerelationships WHERE isactive=1'): + source = h.ids.get('asset', r['machineid']) + target = h.ids.get('asset', r['related_machineid']) + typeid = h.ids.get('relationshiptype', r['relationshiptypeid']) + if not (source and target and typeid) or source == target: + dropped += 1 + continue + key = (source, target, typeid) + if key in seen: + continue + seen.add(key) + status, _ = h.post('/api/assets/relationships', + {'sourceassetid': source, 'targetassetid': target, + 'relationshiptypeid': typeid}) + if status in (200, 201): + made += 1 + return {'relationships': made, 'dropped_unresolved': dropped} + + +def stage_subnets(h): + """Subnets + VLANs. Classic cidr is the suffix only; full CIDR = + INET_NTOA(ipstart)+suffix. VLANs are lookup-or-create by number; duplicate + full CIDRs are first-wins (409 skip).""" + import socket + import struct + subtypes = {s['subnettypeid']: s['subnettype'] for s in + h.source.rows('shopdb_src', 'SELECT subnettypeid, subnettype FROM subnettypes')} + made = 0 + for s in h.source.rows('shopdb_src', + 'SELECT vlan, description, ipstart, cidr, subnettypeid FROM subnets WHERE isactive=1'): + if s['ipstart'] is None or not s['cidr']: + continue + network = socket.inet_ntoa(struct.pack('>I', s['ipstart'] & 0xffffffff)) + suffix = str(s['cidr']).strip() + cidr = network + (suffix if suffix.startswith('/') else '/' + suffix) + vlanid = None + if s['vlan']: + vlanid = _upsert(h, '/api/network/vlans', + {'vlannumber': s['vlan'], 'name': f"VLAN {s['vlan']}"}, + 'vlannumber', 'vlanid', '/api/network/vlans') + name = (s['description'] or '').strip() or cidr + payload = {'name': name, 'cidr': cidr, 'networkaddress': network, + 'vlanid': vlanid, 'subnettype': subtypes.get(s['subnettypeid'])} + status, _ = h.post('/api/network/subnets', payload) + if status in (200, 201): + made += 1 + return {'subnets': made} + + +def stage_usb(h): + """CMMC USB kiosk: devices + the check-in/out log paired into checkouts. + Requires selfhosted mode. cmmc users are directory data, not usb rows.""" + from shopdb.core.models import Setting + from shopdb.extensions import db + Setting.set('usb_directory_mode', 'selfhosted', valuetype='string', category='usb') + db.session.commit() + + devices = 0 + for d in h.source.rows('cmmc_usb_src', 'SELECT device_id, device_desc, device_owner, locker_location FROM devices'): + did = (d['device_id'] or '').strip() + if not did or did.lower() in ('test', 'test123'): + continue + status, _ = h.post('/api/usb', {'device_id': did, 'device_desc': d['device_desc'], + 'device_owner': d['device_owner'], + 'locker_location': d['locker_location']}) + if status in (200, 201, 409): + devices += 1 + + # Replay the log chronologically per device, tracking open state so we only + # check out an available device and check in an out one (the source log has + # unpaired entries - a checkin with no open checkout would 400). + events = 0 + checked_out = {} + log = h.source.rows('cmmc_usb_src', + 'SELECT device_id, badge_number, action, timestamp, locker_location ' + 'FROM checkinoutlog ORDER BY device_id, timestamp') + for e in log: + did = (e['device_id'] or '').strip() + badge = str(e['badge_number'] or '').strip() + action = (e['action'] or '').strip().lower() + if not (did and badge): + continue + when = str(e['timestamp']) if e['timestamp'] else None + if 'out' in action and not checked_out.get(did): + status, _ = h.post(f'/api/usb/{did}/checkout', + {'badge': badge, 'checkouttime': when, + 'locker_location': e['locker_location']}) + if status in (200, 201): + checked_out[did] = True + events += 1 + elif 'in' in action and checked_out.get(did): + status, _ = h.post(f'/api/usb/{did}/checkin', {'badge': badge, 'checkintime': when}) + if status in (200, 201): + checked_out[did] = False + events += 1 + return {'usb_devices': devices, 'usb_events': events} + + +def stage_verify(h): + """Row-count audit: source active count vs target count for each entity.""" + checks = [ + ('vendors', 'shopdb_src', 'SELECT COUNT(*) c FROM vendors WHERE isactive=1', 'vendors'), + ('assets', 'shopdb_src', 'SELECT COUNT(*) c FROM machines', 'assets'), + ('applications', 'shopdb_src', 'SELECT COUNT(*) c FROM applications WHERE isactive=1', 'applications'), + ('knowledgebase', 'shopdb_src', 'SELECT COUNT(*) c FROM knowledgebase WHERE isactive=1', 'knowledgebase'), + ('employees', 'wjf_employees_src', 'SELECT COUNT(*) c FROM employees', 'directoryemployees'), + ] + from shopdb.extensions import db + from sqlalchemy import text + report = {} + for label, sdb, sql, target_table in checks: + src = h.source.rows(sdb, sql)[0]['c'] + tgt = db.session.execute(text(f'SELECT COUNT(*) FROM {target_table}')).scalar() + report[label] = f'source~{src} target={tgt}' + return report + + STAGES = { 'reference': stage_reference, 'employees': stage_employees, 'catalog': stage_catalog, 'assets': stage_assets, + 'locations': stage_locations, 'communications': stage_communications, 'applications': stage_applications, 'warranties': stage_warranties, 'notifications': stage_notifications, 'knowledgebase': stage_knowledgebase, - # TODO: 'locations', 'relationships', 'subnets', 'usb', 'verify' + 'relationships': stage_relationships, + 'subnets': stage_subnets, + 'usb': stage_usb, + 'verify': stage_verify, } def main(): parser = argparse.ArgumentParser() - parser.add_argument('--stages', default='reference,employees,catalog,assets', - help='comma list of stages to run') + parser.add_argument( + '--stages', + default='reference,employees,catalog,assets,locations,communications,' + 'applications,warranties,notifications,knowledgebase,' + 'relationships,subnets,usb,verify', + help='comma list of stages to run') args = parser.parse_args() h = Harness()