WJ import loader: dependent-entity stages (comms, apps/installs, warranty, notif, KB)
Some checks failed
CI / backend (push) Successful in 1m38s
CI / naming (push) Successful in 1s
CI / frontend (push) Successful in 8s
CI / migrations-mysql (push) Failing after 8s

All consume the machineid->assetid crosswalk from the assets hub. Verified
against the scratch target, zero endpoint errors:

- communications: 435 primary IPs folded onto assets. No bulk endpoint exists,
  so this is the plan's documented direct-ORM gap (reads the source
  communications table where comstypeid=1 AND isprimary=1, not machines.ipaddress1
  which is empty).
- applications: supportteams 45, applications 121 (colliding names dedup via the
  unique-appname 409-resolve), appversions 47, installs 653 (machineid ->
  assetid -> computerid; only computer assets take installs).
- warranties: 424 linked, vendor hardcoded Dell (source has none).
- notifications: types 6, notifications 261 (2099 sentinel endtime clamped).
- knowledgebase: 341 (appid resolved through the applications name map).

Inactive rows skipped everywhere per the decisions. Remaining loader stages:
locations (the 158 LocationOnly rows), relationships (301 active edges),
subnets/VLANs, usb (cmmc pairing), verify.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
cproudlock
2026-07-13 11:43:02 -04:00
parent 6c4bd20e01
commit 6d79e469fa

View File

@@ -302,12 +302,193 @@ def stage_assets(h):
return counts
def stage_communications(h):
"""Fold each asset's primary IP from the source communications table. There
is no bulk-communications endpoint, so this is one of the plan's documented
direct-ORM gaps (the API only takes a primary IP on asset create/update)."""
from shopdb.core.models import Communication, CommunicationType
from shopdb.extensions import db
ip_type = CommunicationType.query.filter_by(comtype='IP').first()
rows = h.source.rows('shopdb_src',
'SELECT machineid, address FROM communications '
'WHERE comstypeid=1 AND isprimary=1')
added = 0
for r in rows:
assetid = h.ids.get('asset', r['machineid'])
ip = (r['address'] or '').strip()
if not (assetid and ip and ip_type):
continue
if Communication.query.filter_by(assetid=assetid, isprimary=True).first():
continue
db.session.add(Communication(assetid=assetid, comtypeid=ip_type.comtypeid,
ipaddress=ip, isprimary=True))
added += 1
db.session.commit()
return {'primary_ips': added}
def stage_applications(h):
"""Support teams + applications catalog + versions + installs. Skips
inactive rows (decision); dedup of colliding app names is automatic via the
unique-appname 409-resolve."""
from plugins.computers.models import Computer
counts = {}
for t in h.source.rows('shopdb_src',
'SELECT supporteamid, teamname, teamurl FROM supportteams WHERE isactive=1'):
name = (t['teamname'] or '').strip()
if not name:
continue
newid = _upsert(h, '/api/supportteams', {'teamname': name, 'teamurl': t['teamurl']},
'teamname', 'supportteamid', '/api/supportteams')
if newid:
h.ids.put('supportteam', t['supporteamid'], newid)
counts['supportteams'] = h.ids.count('supportteam')
for a in h.source.rows('shopdb_src',
'SELECT appid, appname, appdescription, supportteamid, '
'isinstallable, applicationnotes, installpath FROM applications '
'WHERE isactive=1'):
name = (a['appname'] or '').strip()
if not name:
continue
payload = {'appname': name, 'appdescription': a['appdescription'],
'supportteamid': h.ids.get('supportteam', a['supportteamid']),
'isinstallable': bool(a['isinstallable']),
'applicationnotes': a['applicationnotes'], 'installpath': a['installpath']}
newid = _upsert(h, '/api/applications', payload, 'appname', 'appid',
'/api/applications')
if newid:
h.ids.put('app', a['appid'], newid)
counts['applications'] = h.ids.count('app')
for v in h.source.rows('shopdb_src',
'SELECT appversionid, appid, version, releasedate, notes, dateadded '
'FROM appversions WHERE isactive=1'):
appid = h.ids.get('app', v['appid'])
ver = (v['version'] or '').strip()
if not (appid and ver):
continue
status, data = h.post(f'/api/applications/{appid}/versions',
{'version': ver, 'releasedate': str(v['releasedate']) if v['releasedate'] else None,
'notes': v['notes'], 'dateadded': str(v['dateadded']) if v['dateadded'] else None})
if status in (200, 201):
h.ids.put('appversion', v['appversionid'], _id_of(data, 'appversionid'))
counts['appversions'] = h.ids.count('appversion')
installed = 0
for i in h.source.rows('shopdb_src',
'SELECT appid, appversionid, machineid FROM installedapps WHERE isactive=1'):
assetid = h.ids.get('asset', i['machineid'])
appid = h.ids.get('app', i['appid'])
if not (assetid and appid):
continue
computer = Computer.query.filter_by(assetid=assetid).first()
if not computer:
continue
payload = {'appid': appid}
vid = h.ids.get('appversion', i['appversionid'])
if vid:
payload['appversionid'] = vid
status, _ = h.post(f'/api/computers/{computer.computerid}/apps', payload)
if status in (200, 201):
installed += 1
counts['installs'] = installed
return counts
def stage_warranties(h):
"""Warranties -> POST /api/warranty, linked to the asset via the crosswalk.
The source has no vendor; hardcode Dell (decision)."""
linked = 0
for w in h.source.rows('shopdb_src',
'SELECT machineid, warrantyname, enddate, servicelevel, dateadded '
'FROM warranties'):
assetid = h.ids.get('asset', w['machineid'])
if not assetid:
continue
payload = {'vendor': 'Dell', 'provider': 'manual',
'servicelevel': (w['servicelevel'] or '').strip() or None,
'enddate': str(w['enddate']) if w['enddate'] else None,
'assetids': [assetid],
'dateadded': str(w['dateadded']) if w['dateadded'] else None}
status, _ = h.post('/api/warranty', payload)
if status in (200, 201):
linked += 1
return {'warranties': linked}
def stage_notifications(h):
"""Notification types + notifications."""
counts = {}
for t in h.source.rows('shopdb_src',
'SELECT notificationtypeid, typename, typecolor FROM notificationtypes WHERE isactive=1'):
name = (t['typename'] or '').strip()
if not name:
continue
newid = _upsert(h, '/api/notifications/types',
{'typename': name, 'typecolor': (t['typecolor'] or '').strip() or None},
'typename', 'notificationtypeid', '/api/notifications/types')
if newid:
h.ids.put('notificationtype', t['notificationtypeid'], newid)
counts['notificationtypes'] = h.ids.count('notificationtype')
made = 0
for n in h.source.rows('shopdb_src',
'SELECT notificationtypeid, businessunitid, appid, notification, '
'starttime, endtime, ticketnumber, link, isshopfloor, employeesso '
'FROM notifications WHERE isactive=1'):
text = (n['notification'] or '').strip()
if not text:
continue
endtime = str(n['endtime']) if n['endtime'] else None
if endtime and endtime.startswith('2099'):
endtime = None
payload = {'notification': text,
'notificationtypeid': h.ids.get('notificationtype', n['notificationtypeid']),
'businessunitid': h.ids.get('businessunit', n['businessunitid']),
'appid': h.ids.get('app', n['appid']),
'starttime': str(n['starttime']) if n['starttime'] else None,
'endtime': endtime, 'ticketnumber': n['ticketnumber'], 'link': n['link'],
'isshopfloor': bool(n['isshopfloor']), 'employeesso': n['employeesso']}
status, _ = h.post('/api/notifications', payload)
if status in (200, 201):
made += 1
counts['notifications'] = made
return counts
def stage_knowledgebase(h):
"""KB articles. appid resolves through the applications name map (topics
folded into applications)."""
made = 0
for k in h.source.rows('shopdb_src',
'SELECT shortdescription, keywords, appid, linkurl, notes '
'FROM knowledgebase WHERE isactive=1'):
title = (k['shortdescription'] or '').strip()
url = (k['linkurl'] or '').strip()
if not (title and url):
continue
payload = {'shortdescription': title, 'linkurl': url,
'keywords': k['keywords'], 'notes': k['notes'],
'appid': h.ids.get('app', k['appid'])}
status, _ = h.post('/api/knowledgebase', payload)
if status in (200, 201):
made += 1
return {'kb_articles': made}
STAGES = {
'reference': stage_reference,
'employees': stage_employees,
'catalog': stage_catalog,
'assets': stage_assets,
# TODO: 'applications', 'dependents', 'network-subnets', 'usb', 'verify'
'communications': stage_communications,
'applications': stage_applications,
'warranties': stage_warranties,
'notifications': stage_notifications,
'knowledgebase': stage_knowledgebase,
# TODO: 'locations', 'relationships', 'subnets', 'usb', 'verify'
}