ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retire stale legacy-import mappers; add adoption playbook
Some checks failed
CI / naming (push) Has been cancelled
Details
CI / frontend (push) Has been cancelled
Details
CI / migrations-mysql (push) Has been cancelled
Details
CI / backend (push) Has been cancelled
Details

Browse Source 
Cleanup after the reference loader (scripts/site_imports/wjf/) proved out.

Removed the drifted direct-SQL migrators - migrate_assets/communications/
notifications/usb.py, run_migration.py, verify_migration.py, and
scripts/import_from_mysql.py. They targeted a nonexistent equipment table, the
retired Machine model, and columns that no longer exist; nothing imported them.
scripts/migration/README.md now points at the import API + the site loader.
Kept the one-time SQL fixups (fix_legacy_schema.sql, one-offs/).

Added docs/IMPORT-ADOPTION.md: the two-layer import story (stable IMPORT-API
contract + per-site loader), stage-ordering + crosswalk guidance, the
agent-assisted mapping path, and what the WJ loader demonstrates. Updated the
loader README to complete status (all 15 stages, final counts).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
cproudlock
2026-07-13 12:18:14 -04:00
parent af9afc15c5
commit 474f245ae7
7 changed files with 0 additions and 1630 deletions
Download Patch File Download Diff File Expand all files Collapse all files

581
scripts/import_from_mysql.py
View File

@@ -1,581 +0,0 @@
#!/usr/bin/env python3
"""
Import data from legacy MySQL ShopDB to new Flask ShopDB.
Usage:
cd /home/camp/projects/shopdb-flask
source venv/bin/activate
python scripts/import_from_mysql.py
"""
import os
import sys
import pymysql
from datetime import datetime
from dotenv import load_dotenv
# Add parent directory to path for imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Load environment variables
load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.env'))
# MySQL connection settings
MYSQL_CONFIG = {
'host': '127.0.0.1',
'port': 3306,
'user': 'root',
'password': 'rootpassword',
'database': 'shopdb',
'charset': 'utf8mb4',
'cursorclass': pymysql.cursors.DictCursor
}
def get_mysql_connection():
"""Get MySQL connection."""
return pymysql.connect(**MYSQL_CONFIG)
def import_vendors(mysql_conn, db, Vendor):
"""Import vendors from MySQL."""
print("Importing vendors...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM vendors WHERE isactive = 1")
vendors = cursor.fetchall()
count = 0
for v in vendors:
existing = Vendor.query.filter_by(vendor=v['vendor']).first()
if not existing:
vendor = Vendor(
vendor=v['vendor'],
isactive=True
)
db.session.add(vendor)
count += 1
db.session.commit()
print(f" Imported {count} vendors")
return count
def import_machinetypes(mysql_conn, db, MachineType):
"""Import machine types from MySQL with category mapping."""
print("Importing machine types...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM machinetypes WHERE isactive = 1")
types = cursor.fetchall()
# Category mapping based on machinetype name
pc_types = ['PC']
network_types = ['Access Point', 'IDF', 'Switch', 'Server', 'Camera']
printer_types = ['Printer']
count = 0
for t in types:
existing = MachineType.query.filter_by(machinetype=t['machinetype']).first()
if not existing:
# Determine category
if t['machinetype'] in pc_types:
category = 'PC'
elif t['machinetype'] in network_types:
category = 'Network'
elif t['machinetype'] in printer_types:
category = 'Printer'
else:
category = 'Equipment'
mt = MachineType(
machinetype=t['machinetype'],
category=category,
description=t.get('machinedescription'),
isactive=True
)
db.session.add(mt)
count += 1
db.session.commit()
print(f" Imported {count} machine types")
return count
def import_pctypes(mysql_conn, db, PCType):
"""Import PC types from MySQL."""
print("Importing PC types...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM pctype WHERE isactive = '1'")
types = cursor.fetchall()
count = 0
for t in types:
existing = PCType.query.filter_by(pctype=t['typename']).first()
if not existing:
pctype = PCType(
pctype=t['typename'],
description=t.get('description'),
isactive=True
)
db.session.add(pctype)
count += 1
db.session.commit()
print(f" Imported {count} PC types")
return count
def import_businessunits(mysql_conn, db, BusinessUnit):
"""Import business units from MySQL."""
print("Importing business units...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM businessunits WHERE isactive = 1")
units = cursor.fetchall()
count = 0
for bu in units:
existing = BusinessUnit.query.filter_by(businessunit=bu['businessunit']).first()
if not existing:
unit = BusinessUnit(
businessunit=bu['businessunit'],
isactive=True
)
db.session.add(unit)
count += 1
db.session.commit()
print(f" Imported {count} business units")
return count
def import_statuses(mysql_conn, db, MachineStatus):
"""Import machine statuses from MySQL."""
print("Importing machine statuses...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM machinestatus WHERE isactive = 1")
statuses = cursor.fetchall()
count = 0
for s in statuses:
existing = MachineStatus.query.filter_by(status=s['machinestatus']).first()
if not existing:
status = MachineStatus(
status=s['machinestatus'],
description=s.get('statusdescription'),
isactive=True
)
db.session.add(status)
count += 1
db.session.commit()
print(f" Imported {count} statuses")
return count
def import_operatingsystems(mysql_conn, db, OperatingSystem):
"""Import operating systems from MySQL."""
print("Importing operating systems...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM operatingsystems")
os_list = cursor.fetchall()
count = 0
for os_item in os_list:
os_name = os_item.get('operatingsystem') or os_item.get('osname')
if not os_name:
continue
existing = OperatingSystem.query.filter_by(osname=os_name).first()
if not existing:
os_obj = OperatingSystem(
osname=os_name,
osversion=os_item.get('osversion'),
isactive=True
)
db.session.add(os_obj)
count += 1
db.session.commit()
print(f" Imported {count} operating systems")
return count
def import_models(mysql_conn, db, Model, Vendor, MachineType):
"""Import models from MySQL."""
print("Importing models...")
cursor = mysql_conn.cursor()
cursor.execute("""
SELECT m.*, v.vendor as vendor_name, mt.machinetype as type_name
FROM models m
LEFT JOIN vendors v ON m.vendorid = v.vendorid
LEFT JOIN machinetypes mt ON m.machinetypeid = mt.machinetypeid
WHERE m.isactive = 1
""")
models = cursor.fetchall()
count = 0
for m in models:
existing = Model.query.filter_by(modelnumber=m['modelnumber']).first()
if not existing:
# Find vendor and machinetype in new db
vendor = Vendor.query.filter_by(vendor=m['vendor_name']).first() if m['vendor_name'] else None
machinetype = MachineType.query.filter_by(machinetype=m['type_name']).first() if m['type_name'] else None
model = Model(
modelnumber=m['modelnumber'],
vendorid=vendor.vendorid if vendor else None,
machinetypeid=machinetype.machinetypeid if machinetype else None,
notes=m.get('notes'),
isactive=True
)
db.session.add(model)
count += 1
db.session.commit()
print(f" Imported {count} models")
return count
def import_relationshiptypes(mysql_conn, db, RelationshipType):
"""Import relationship types from MySQL."""
print("Importing relationship/connection types...")
cursor = mysql_conn.cursor()
cursor.execute("SELECT * FROM relationshiptypes WHERE isactive = 1")
types = cursor.fetchall()
count = 0
for rt in types:
existing = RelationshipType.query.filter_by(relationshiptype=rt['relationshiptype']).first()
if not existing:
rel_type = RelationshipType(
relationshiptype=rt['relationshiptype'],
description=rt.get('description'),
isactive=True
)
db.session.add(rel_type)
count += 1
db.session.commit()
print(f" Imported {count} relationship types")
return count
def import_machines(mysql_conn, db, Machine, MachineType, MachineStatus,
Vendor, Model, BusinessUnit, OperatingSystem, Location,
Communication, CommunicationType, PCType):
"""Import machines (Equipment and PCs) from MySQL."""
print("Importing machines...")
cursor = mysql_conn.cursor()
# Get machines with related data
cursor.execute("""
SELECT m.*,
mt.machinetype as type_name,
ms.machinestatus as status_name,
v.vendor as vendor_name,
mdl.modelnumber as model_name,
bu.businessunit as bu_name,
os.operatingsystem as os_name,
pt.typename as pctype_name
FROM machines m
LEFT JOIN machinetypes mt ON m.machinetypeid = mt.machinetypeid
LEFT JOIN machinestatus ms ON m.machinestatusid = ms.machinestatusid
LEFT JOIN models mdl ON m.modelnumberid = mdl.modelnumberid
LEFT JOIN vendors v ON mdl.vendorid = v.vendorid
LEFT JOIN businessunits bu ON m.businessunitid = bu.businessunitid
LEFT JOIN operatingsystems os ON m.osid = os.osid
LEFT JOIN pctype pt ON m.pctypeid = pt.pctypeid
WHERE m.isactive = 1
""")
machines = cursor.fetchall()
# Get or create IP communication type
ip_comtype = CommunicationType.query.filter_by(comtype='IP').first()
if not ip_comtype:
ip_comtype = CommunicationType(comtype='IP', description='IP Network')
db.session.add(ip_comtype)
db.session.flush()
# Build lookup maps
type_map = {t.machinetype: t for t in MachineType.query.all()}
status_map = {s.status: s for s in MachineStatus.query.all()}
vendor_map = {v.vendor: v for v in Vendor.query.all()}
model_map = {m.modelnumber: m for m in Model.query.all()}
bu_map = {b.businessunit: b for b in BusinessUnit.query.all()}
os_map = {o.osname: o for o in OperatingSystem.query.all()}
pctype_map = {p.pctype: p for p in PCType.query.all()}
# Track old->new ID mapping for relationships
machine_id_map = {}
count = 0
comm_count = 0
skipped = 0
for m in machines:
# Skip machines without a machinenumber
if not m.get('machinenumber'):
skipped += 1
continue
# Check if already exists
existing = Machine.query.filter_by(machinenumber=m['machinenumber']).first()
if existing:
machine_id_map[m['machineid']] = existing.machineid
continue
# Get related objects
machinetype = type_map.get(m['type_name'])
status = status_map.get(m['status_name'])
vendor = vendor_map.get(m['vendor_name'])
model = model_map.get(m['model_name'])
bu = bu_map.get(m['bu_name'])
os_obj = os_map.get(m['os_name'])
pctype = pctype_map.get(m['pctype_name'])
machine = Machine(
machinenumber=m['machinenumber'],
alias=m.get('alias'),
hostname=m.get('hostname'),
serialnumber=m.get('serialnumber'),
machinetypeid=machinetype.machinetypeid if machinetype else None,
pctypeid=pctype.pctypeid if pctype else None,
statusid=status.statusid if status else None,
vendorid=vendor.vendorid if vendor else None,
modelnumberid=model.modelnumberid if model else None,
businessunitid=bu.businessunitid if bu else None,
osid=os_obj.osid if os_obj else None,
mapleft=m.get('mapleft'),
maptop=m.get('maptop'),
isvnc=bool(m.get('isvnc')),
iswinrm=bool(m.get('iswinrm')),
islocationonly=bool(m.get('islocationonly')),
loggedinuser=m.get('loggedinuser'),
notes=m.get('machinenotes'),
isactive=True
)
db.session.add(machine)
db.session.flush() # Get the new ID
machine_id_map[m['machineid']] = machine.machineid
count += 1
# Import IP addresses
if m.get('ipaddress1'):
comm = Communication(
machineid=machine.machineid,
comtypeid=ip_comtype.comtypeid,
ipaddress=m['ipaddress1'],
isprimary=True
)
db.session.add(comm)
comm_count += 1
if m.get('ipaddress2'):
comm = Communication(
machineid=machine.machineid,
comtypeid=ip_comtype.comtypeid,
ipaddress=m['ipaddress2'],
isprimary=False
)
db.session.add(comm)
comm_count += 1
db.session.commit()
print(f" Imported {count} machines with {comm_count} IP addresses (skipped {skipped} invalid)")
return machine_id_map
def import_relationships(mysql_conn, db, MachineRelationship, RelationshipType, machine_id_map):
"""Import machine relationships from MySQL."""
print("Importing machine relationships...")
cursor = mysql_conn.cursor()
cursor.execute("""
SELECT mr.*, rt.relationshiptype
FROM machinerelationships mr
JOIN relationshiptypes rt ON mr.relationshiptypeid = rt.relationshiptypeid
WHERE mr.isactive = 1
""")
relationships = cursor.fetchall()
# Build relationship type map
type_map = {t.relationshiptype: t for t in RelationshipType.query.all()}
count = 0
skipped = 0
for r in relationships:
# Map old IDs to new IDs
parent_id = machine_id_map.get(r['machineid'])
child_id = machine_id_map.get(r['related_machineid'])
if not parent_id or not child_id:
skipped += 1
continue
rel_type = type_map.get(r['relationshiptype'])
if not rel_type:
skipped += 1
continue
# Check if already exists
existing = MachineRelationship.query.filter_by(
parentmachineid=parent_id,
childmachineid=child_id,
relationshiptypeid=rel_type.relationshiptypeid
).first()
if not existing:
relationship = MachineRelationship(
parentmachineid=parent_id,
childmachineid=child_id,
relationshiptypeid=rel_type.relationshiptypeid,
notes=r.get('relationship_notes')
)
db.session.add(relationship)
count += 1
db.session.commit()
print(f" Imported {count} relationships (skipped {skipped})")
return count
def import_printers(mysql_conn, db, Machine, MachineType, Model, Vendor,
Communication, CommunicationType):
"""Import printers from MySQL."""
print("Importing printers...")
# First, ensure we have a Printer machine type
printer_type = MachineType.query.filter_by(machinetype='Printer').first()
if not printer_type:
printer_type = MachineType(machinetype='Printer', category='Printer')
db.session.add(printer_type)
db.session.flush()
# Get or create IP communication type
ip_comtype = CommunicationType.query.filter_by(comtype='IP').first()
if not ip_comtype:
ip_comtype = CommunicationType(comtype='IP', description='IP Network')
db.session.add(ip_comtype)
db.session.flush()
cursor = mysql_conn.cursor()
cursor.execute("""
SELECT p.*, m.modelnumber, v.vendor as vendor_name
FROM printers p
LEFT JOIN models m ON p.modelid = m.modelnumberid
LEFT JOIN vendors v ON m.vendorid = v.vendorid
WHERE p.isactive = 1
""")
printers = cursor.fetchall()
# Build lookup maps
model_map = {m.modelnumber: m for m in Model.query.all()}
vendor_map = {v.vendor: v for v in Vendor.query.all()}
count = 0
comm_count = 0
for p in printers:
# Use windows name as machine number
machine_number = p.get('printerwindowsname') or f"Printer_{p['printerid']}"
existing = Machine.query.filter_by(machinenumber=machine_number).first()
if existing:
continue
model = model_map.get(p.get('modelnumber'))
vendor = vendor_map.get(p.get('vendor_name'))
machine = Machine(
machinenumber=machine_number,
alias=p.get('printercsfname'),
hostname=p.get('fqdn'),
serialnumber=p.get('serialnumber'),
machinetypeid=printer_type.machinetypeid,
vendorid=vendor.vendorid if vendor else None,
modelnumberid=model.modelnumberid if model else None,
mapleft=p.get('mapleft'),
maptop=p.get('maptop'),
notes=p.get('printernotes'),
isactive=True
)
db.session.add(machine)
db.session.flush()
count += 1
# Import IP address
if p.get('ipaddress'):
comm = Communication(
machineid=machine.machineid,
comtypeid=ip_comtype.comtypeid,
ipaddress=p['ipaddress'],
isprimary=True
)
db.session.add(comm)
comm_count += 1
db.session.commit()
print(f" Imported {count} printers with {comm_count} IP addresses")
return count
def main():
"""Main import function."""
print("=" * 60)
print("ShopDB MySQL to Flask Migration")
print("=" * 60)
# Initialize Flask app
from shopdb import create_app
from shopdb.extensions import db
from shopdb.core.models import (
Machine, MachineType, MachineStatus, Vendor, Model,
BusinessUnit, OperatingSystem, Location, PCType
)
from shopdb.core.models.communication import Communication, CommunicationType
from shopdb.core.models.relationship import MachineRelationship, RelationshipType
app = create_app()
with app.app_context():
# Connect to MySQL
print("\nConnecting to MySQL...")
mysql_conn = get_mysql_connection()
print(" Connected!")
try:
# Import reference data
print("\n--- Reference Data ---")
import_vendors(mysql_conn, db, Vendor)
import_machinetypes(mysql_conn, db, MachineType)
import_pctypes(mysql_conn, db, PCType)
import_businessunits(mysql_conn, db, BusinessUnit)
import_statuses(mysql_conn, db, MachineStatus)
import_operatingsystems(mysql_conn, db, OperatingSystem)
import_models(mysql_conn, db, Model, Vendor, MachineType)
import_relationshiptypes(mysql_conn, db, RelationshipType)
# Import machines
print("\n--- Machines ---")
machine_id_map = import_machines(
mysql_conn, db, Machine, MachineType, MachineStatus,
Vendor, Model, BusinessUnit, OperatingSystem, Location,
Communication, CommunicationType, PCType
)
# Import relationships
print("\n--- Relationships ---")
import_relationships(mysql_conn, db, MachineRelationship, RelationshipType, machine_id_map)
# Import printers
print("\n--- Printers ---")
import_printers(mysql_conn, db, Machine, MachineType, Model, Vendor,
Communication, CommunicationType)
print("\n" + "=" * 60)
print("Import complete!")
print("=" * 60)
finally:
mysql_conn.close()
if __name__ == '__main__':
main()

285
scripts/migration/migrate_assets.py
View File

@@ -1,285 +0,0 @@
"""
Migrate machines table to assets + extension tables.
This script migrates data from the legacy machines table to the new
Asset architecture with plugin-owned extension tables.
Strategy:
1. Preserve IDs: assets.assetid = original machines.machineid
2. Create asset record, then type-specific extension record
3. Map machine types to asset types:
- MachineType = Equipment -> equipment extension
- MachineType = PC -> computers extension
- MachineType = Network/Camera/etc -> network_devices extension
- Printers -> handled separately by printers plugin
Usage:
python -m scripts.migration.migrate_assets --source <connection_string>
"""
import argparse
import logging
from datetime import datetime
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def get_machine_type_mapping():
"""Map legacy machine type IDs to asset types."""
return {
# Equipment types
'CNC': 'equipment',
'CMM': 'equipment',
'Lathe': 'equipment',
'Grinder': 'equipment',
'EDM': 'equipment',
'Mill': 'equipment',
'Press': 'equipment',
'Robot': 'equipment',
'Part Marker': 'equipment',
# PC types
'PC': 'computer',
'Workstation': 'computer',
'Laptop': 'computer',
'Server': 'computer',
# Network types
'Switch': 'network_device',
'Router': 'network_device',
'Access Point': 'network_device',
'Camera': 'network_device',
'IDF': 'network_device',
'MDF': 'network_device',
'Firewall': 'network_device',
}
def migrate_machine_to_asset(machine_row, asset_type_id, target_session):
"""
Create an Asset record from a Machine record.
Args:
machine_row: Row from source machines table
asset_type_id: Target asset type ID
target_session: SQLAlchemy session for target database
Returns:
Created asset ID
"""
# Insert into assets table
target_session.execute(text("""
INSERT INTO assets (
assetid, assetnumber, name, serialnumber,
assettypeid, statusid, locationid, businessunitid,
mapx, mapy, notes, isactive, createddate, modifieddate
) VALUES (
:assetid, :assetnumber, :name, :serialnumber,
:assettypeid, :statusid, :locationid, :businessunitid,
:mapx, :mapy, :notes, :isactive, :createddate, :modifieddate
)
"""), {
'assetid': machine_row['machineid'],
'assetnumber': machine_row['machinenumber'],
'name': machine_row.get('alias'),
'serialnumber': machine_row.get('serialnumber'),
'assettypeid': asset_type_id,
'statusid': machine_row.get('statusid', 1),
'locationid': machine_row.get('locationid'),
'businessunitid': machine_row.get('businessunitid'),
'mapx': machine_row.get('mapx'),
'mapy': machine_row.get('mapy'),
'notes': machine_row.get('notes'),
'isactive': machine_row.get('isactive', True),
'createddate': machine_row.get('createddate', datetime.utcnow()),
'modifieddate': machine_row.get('modifieddate', datetime.utcnow()),
})
return machine_row['machineid']
def migrate_equipment(machine_row, asset_id, target_session):
"""Create equipment extension record."""
target_session.execute(text("""
INSERT INTO equipment (
assetid, equipmenttypeid, vendorid, modelnumberid,
requiresmanualconfig, islocationonly, isactive, createddate
) VALUES (
:assetid, :equipmenttypeid, :vendorid, :modelnumberid,
:requiresmanualconfig, :islocationonly, :isactive, :createddate
)
"""), {
'assetid': asset_id,
'equipmenttypeid': machine_row.get('machinetypeid'), # May need mapping
'vendorid': machine_row.get('vendorid'),
'modelnumberid': machine_row.get('modelnumberid'),
'requiresmanualconfig': machine_row.get('requiresmanualconfig', False),
'islocationonly': machine_row.get('islocationonly', False),
'isactive': True,
'createddate': datetime.utcnow(),
})
def migrate_computer(machine_row, asset_id, target_session):
"""Create computer extension record."""
target_session.execute(text("""
INSERT INTO computers (
assetid, computertypeid, vendorid, operatingsystemid,
hostname, currentuserid, lastuserid, lastboottime,
lastzabbixsync, isvnc, isactive, createddate
) VALUES (
:assetid, :computertypeid, :vendorid, :operatingsystemid,
:hostname, :currentuserid, :lastuserid, :lastboottime,
:lastzabbixsync, :isvnc, :isactive, :createddate
)
"""), {
'assetid': asset_id,
'computertypeid': machine_row.get('pctypeid'),
'vendorid': machine_row.get('vendorid'),
'operatingsystemid': machine_row.get('operatingsystemid'),
'hostname': machine_row.get('hostname'),
'currentuserid': machine_row.get('currentuserid'),
'lastuserid': machine_row.get('lastuserid'),
'lastboottime': machine_row.get('lastboottime'),
'lastzabbixsync': machine_row.get('lastzabbixsync'),
'isvnc': machine_row.get('isvnc', False),
'isactive': True,
'createddate': datetime.utcnow(),
})
def migrate_network_device(machine_row, asset_id, target_session):
"""Create network device extension record."""
target_session.execute(text("""
INSERT INTO networkdevices (
assetid, networkdevicetypeid, vendorid, hostname,
firmwareversion, portcount, ispoe, ismanaged,
isactive, createddate
) VALUES (
:assetid, :networkdevicetypeid, :vendorid, :hostname,
:firmwareversion, :portcount, :ispoe, :ismanaged,
:isactive, :createddate
)
"""), {
'assetid': asset_id,
'networkdevicetypeid': machine_row.get('machinetypeid'), # May need mapping
'vendorid': machine_row.get('vendorid'),
'hostname': machine_row.get('hostname'),
'firmwareversion': machine_row.get('firmwareversion'),
'portcount': machine_row.get('portcount'),
'ispoe': machine_row.get('ispoe', False),
'ismanaged': machine_row.get('ismanaged', False),
'isactive': True,
'createddate': datetime.utcnow(),
})
def run_migration(source_conn_str, target_conn_str, dry_run=False):
"""
Run the full migration from machines to assets.
Args:
source_conn_str: Connection string for source (VBScript) database
target_conn_str: Connection string for target (Flask) database
dry_run: If True, don't commit changes
"""
source_engine = create_engine(source_conn_str)
target_engine = create_engine(target_conn_str)
SourceSession = sessionmaker(bind=source_engine)
TargetSession = sessionmaker(bind=target_engine)
source_session = SourceSession()
target_session = TargetSession()
try:
# Get asset type mappings from target database
asset_types = {}
result = target_session.execute(text("SELECT assettypeid, assettype FROM assettypes"))
for row in result:
asset_types[row.assettype] = row.assettypeid
# Get machine type to asset type mapping
type_mapping = get_machine_type_mapping()
# Fetch all machines from source
machines = source_session.execute(text("""
SELECT m.*, mt.machinetype
FROM machines m
LEFT JOIN machinetypes mt ON m.machinetypeid = mt.machinetypeid
"""))
migrated = 0
errors = 0
for machine in machines:
machine_dict = dict(machine._mapping)
try:
# Determine asset type
machine_type_name = machine_dict.get('machinetype', '')
asset_type_name = type_mapping.get(machine_type_name, 'equipment')
asset_type_id = asset_types.get(asset_type_name)
if not asset_type_id:
logger.warning(f"Unknown asset type for machine {machine_dict['machineid']}: {machine_type_name}")
errors += 1
continue
# Create asset record
asset_id = migrate_machine_to_asset(machine_dict, asset_type_id, target_session)
# Create extension record based on type
if asset_type_name == 'equipment':
migrate_equipment(machine_dict, asset_id, target_session)
elif asset_type_name == 'computer':
migrate_computer(machine_dict, asset_id, target_session)
elif asset_type_name == 'network_device':
migrate_network_device(machine_dict, asset_id, target_session)
migrated += 1
if migrated % 100 == 0:
logger.info(f"Migrated {migrated} machines...")
except Exception as e:
logger.error(f"Error migrating machine {machine_dict.get('machineid')}: {e}")
errors += 1
if dry_run:
logger.info("Dry run - rolling back changes")
target_session.rollback()
else:
target_session.commit()
logger.info(f"Migration complete: {migrated} migrated, {errors} errors")
finally:
source_session.close()
target_session.close()
def main():
parser = argparse.ArgumentParser(description='Migrate machines to assets')
parser.add_argument('--source', required=True, help='Source database connection string')
parser.add_argument('--target', help='Target database connection string (default: app config)')
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
args = parser.parse_args()
target = args.target
if not target:
# Load from Flask config
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
from shopdb import create_app
app = create_app()
target = app.config['SQLALCHEMY_DATABASE_URI']
run_migration(args.source, target, args.dry_run)
if __name__ == '__main__':
main()

113
scripts/migration/migrate_communications.py
View File

@@ -1,113 +0,0 @@
"""
Migrate communications table to use assetid instead of machineid.
This script updates the communications table FK from machineid to assetid.
Since assetid matches the original machineid, this is mostly a schema update.
Usage:
python -m scripts.migration.migrate_communications
"""
import logging
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def run_migration(conn_str, dry_run=False):
"""
Update communications to use assetid.
Args:
conn_str: Database connection string
dry_run: If True, don't commit changes
"""
engine = create_engine(conn_str)
Session = sessionmaker(bind=engine)
session = Session()
try:
# Check if assetid column already exists
result = session.execute(text("""
SELECT COUNT(*) FROM information_schema.columns
WHERE table_name = 'communications' AND column_name = 'assetid'
"""))
has_assetid = result.scalar() > 0
if not has_assetid:
logger.info("Adding assetid column to communications table...")
# Add assetid column
session.execute(text("""
ALTER TABLE communications
ADD COLUMN assetid INT NULL
"""))
# Copy machineid values to assetid
session.execute(text("""
UPDATE communications
SET assetid = machineid
WHERE machineid IS NOT NULL
"""))
# Add FK constraint (optional, depends on DB)
try:
session.execute(text("""
ALTER TABLE communications
ADD CONSTRAINT fk_comm_asset
FOREIGN KEY (assetid) REFERENCES assets(assetid)
"""))
except Exception as e:
logger.warning(f"Could not add FK constraint: {e}")
logger.info("assetid column added and populated")
else:
logger.info("assetid column already exists")
# Count records
result = session.execute(text("""
SELECT COUNT(*) FROM communications WHERE assetid IS NOT NULL
"""))
count = result.scalar()
logger.info(f"Communications with assetid: {count}")
if dry_run:
logger.info("Dry run - rolling back changes")
session.rollback()
else:
session.commit()
logger.info("Migration complete")
except Exception as e:
logger.error(f"Migration error: {e}")
session.rollback()
raise
finally:
session.close()
def main():
import argparse
import os
import sys
parser = argparse.ArgumentParser(description='Migrate communications to use assetid')
parser.add_argument('--connection', help='Database connection string')
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
args = parser.parse_args()
conn_str = args.connection
if not conn_str:
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
from shopdb import create_app
app = create_app()
conn_str = app.config['SQLALCHEMY_DATABASE_URI']
run_migration(conn_str, args.dry_run)
if __name__ == '__main__':
main()

139
scripts/migration/migrate_notifications.py
View File

@@ -1,139 +0,0 @@
"""
Migrate notifications from legacy database.
This script migrates notification data from the VBScript database
to the new notifications plugin schema.
Usage:
python -m scripts.migration.migrate_notifications --source <connection_string>
"""
import argparse
import logging
from datetime import datetime
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def get_notification_type_mapping(target_session):
"""Get mapping of type names to IDs in target database."""
result = target_session.execute(text(
"SELECT notificationtypeid, typename FROM notificationtypes"
))
return {row.typename.lower(): row.notificationtypeid for row in result}
def run_migration(source_conn_str, target_conn_str, dry_run=False):
"""
Run notification migration.
Args:
source_conn_str: Source database connection string
target_conn_str: Target database connection string
dry_run: If True, don't commit changes
"""
source_engine = create_engine(source_conn_str)
target_engine = create_engine(target_conn_str)
SourceSession = sessionmaker(bind=source_engine)
TargetSession = sessionmaker(bind=target_engine)
source_session = SourceSession()
target_session = TargetSession()
try:
# Get type mappings
type_mapping = get_notification_type_mapping(target_session)
# Default type if not found
default_type_id = type_mapping.get('general', 1)
# Fetch notifications from source
# Adjust column names based on actual legacy schema
notifications = source_session.execute(text("""
SELECT n.*, nt.typename
FROM notifications n
LEFT JOIN notificationtypes nt ON n.notificationtypeid = nt.notificationtypeid
"""))
migrated = 0
errors = 0
for notif in notifications:
notif_dict = dict(notif._mapping)
try:
# Map notification type
type_name = (notif_dict.get('typename') or 'general').lower()
type_id = type_mapping.get(type_name, default_type_id)
# Insert into target
target_session.execute(text("""
INSERT INTO notifications (
title, message, notificationtypeid,
startdate, enddate, ispinned, showbanner, allday,
linkurl, affectedsystems, isactive, createddate
) VALUES (
:title, :message, :notificationtypeid,
:startdate, :enddate, :ispinned, :showbanner, :allday,
:linkurl, :affectedsystems, :isactive, :createddate
)
"""), {
'title': notif_dict.get('title', 'Untitled'),
'message': notif_dict.get('message', ''),
'notificationtypeid': type_id,
'startdate': notif_dict.get('startdate', datetime.utcnow()),
'enddate': notif_dict.get('enddate'),
'ispinned': notif_dict.get('ispinned', False),
'showbanner': notif_dict.get('showbanner', True),
'allday': notif_dict.get('allday', True),
'linkurl': notif_dict.get('linkurl'),
'affectedsystems': notif_dict.get('affectedsystems'),
'isactive': notif_dict.get('isactive', True),
'createddate': notif_dict.get('createddate', datetime.utcnow()),
})
migrated += 1
except Exception as e:
logger.error(f"Error migrating notification: {e}")
errors += 1
if dry_run:
logger.info("Dry run - rolling back changes")
target_session.rollback()
else:
target_session.commit()
logger.info(f"Migration complete: {migrated} migrated, {errors} errors")
finally:
source_session.close()
target_session.close()
def main():
parser = argparse.ArgumentParser(description='Migrate notifications')
parser.add_argument('--source', required=True, help='Source database connection string')
parser.add_argument('--target', help='Target database connection string')
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
args = parser.parse_args()
target = args.target
if not target:
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
from shopdb import create_app
app = create_app()
target = app.config['SQLALCHEMY_DATABASE_URI']
run_migration(args.source, target, args.dry_run)
if __name__ == '__main__':
main()

199
scripts/migration/migrate_usb.py
View File

@@ -1,199 +0,0 @@
"""
Migrate USB checkout data from legacy database.
This script migrates USB device and checkout data from the VBScript database
to the new USB plugin schema.
Usage:
python -m scripts.migration.migrate_usb --source <connection_string>
"""
import argparse
import logging
from datetime import datetime
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def get_device_type_mapping(target_session):
"""Get mapping of type names to IDs in target database."""
result = target_session.execute(text(
"SELECT usbdevicetypeid, typename FROM usbdevicetypes"
))
return {row.typename.lower(): row.usbdevicetypeid for row in result}
def run_migration(source_conn_str, target_conn_str, dry_run=False):
"""
Run USB device migration.
Args:
source_conn_str: Source database connection string
target_conn_str: Target database connection string
dry_run: If True, don't commit changes
"""
source_engine = create_engine(source_conn_str)
target_engine = create_engine(target_conn_str)
SourceSession = sessionmaker(bind=source_engine)
TargetSession = sessionmaker(bind=target_engine)
source_session = SourceSession()
target_session = TargetSession()
try:
# Get type mappings
type_mapping = get_device_type_mapping(target_session)
default_type_id = type_mapping.get('flash drive', 1)
# Migrate USB devices
# Adjust table/column names based on actual legacy schema
logger.info("Migrating USB devices...")
try:
devices = source_session.execute(text("""
SELECT * FROM usbdevices
"""))
device_id_map = {} # Map old IDs to new IDs
for device in devices:
device_dict = dict(device._mapping)
# Determine device type
type_name = (device_dict.get('typename') or 'flash drive').lower()
type_id = type_mapping.get(type_name, default_type_id)
result = target_session.execute(text("""
INSERT INTO usbdevices (
serialnumber, label, assetnumber, usbdevicetypeid,
capacitygb, vendorid, productid, manufacturer, productname,
ischeckedout, currentuserid, currentusername,
storagelocation, notes, isactive, createddate
) VALUES (
:serialnumber, :label, :assetnumber, :usbdevicetypeid,
:capacitygb, :vendorid, :productid, :manufacturer, :productname,
:ischeckedout, :currentuserid, :currentusername,
:storagelocation, :notes, :isactive, :createddate
)
"""), {
'serialnumber': device_dict.get('serialnumber', f"UNKNOWN_{device_dict.get('usbdeviceid', 0)}"),
'label': device_dict.get('label'),
'assetnumber': device_dict.get('assetnumber'),
'usbdevicetypeid': type_id,
'capacitygb': device_dict.get('capacitygb'),
'vendorid': device_dict.get('vendorid'),
'productid': device_dict.get('productid'),
'manufacturer': device_dict.get('manufacturer'),
'productname': device_dict.get('productname'),
'ischeckedout': device_dict.get('ischeckedout', False),
'currentuserid': device_dict.get('currentuserid'),
'currentusername': device_dict.get('currentusername'),
'storagelocation': device_dict.get('storagelocation'),
'notes': device_dict.get('notes'),
'isactive': device_dict.get('isactive', True),
'createddate': device_dict.get('createddate', datetime.utcnow()),
})
# Get the new ID
new_id = target_session.execute(text("SELECT LAST_INSERT_ID()")).scalar()
device_id_map[device_dict.get('usbdeviceid')] = new_id
logger.info(f"Migrated {len(device_id_map)} USB devices")
except Exception as e:
logger.warning(f"Could not migrate USB devices: {e}")
device_id_map = {}
# Migrate checkout history
logger.info("Migrating USB checkout history...")
try:
checkouts = source_session.execute(text("""
SELECT * FROM usbcheckouts
"""))
checkout_count = 0
for checkout in checkouts:
checkout_dict = dict(checkout._mapping)
old_device_id = checkout_dict.get('usbdeviceid')
new_device_id = device_id_map.get(old_device_id)
if not new_device_id:
logger.warning(f"Skipping checkout - device ID {old_device_id} not found in mapping")
continue
target_session.execute(text("""
INSERT INTO usbcheckouts (
usbdeviceid, userid, username,
checkoutdate, checkindate, expectedreturndate,
purpose, notes, checkedoutby, checkedinby,
isactive, createddate
) VALUES (
:usbdeviceid, :userid, :username,
:checkoutdate, :checkindate, :expectedreturndate,
:purpose, :notes, :checkedoutby, :checkedinby,
:isactive, :createddate
)
"""), {
'usbdeviceid': new_device_id,
'userid': checkout_dict.get('userid', 'unknown'),
'username': checkout_dict.get('username'),
'checkoutdate': checkout_dict.get('checkoutdate', datetime.utcnow()),
'checkindate': checkout_dict.get('checkindate'),
'expectedreturndate': checkout_dict.get('expectedreturndate'),
'purpose': checkout_dict.get('purpose'),
'notes': checkout_dict.get('notes'),
'checkedoutby': checkout_dict.get('checkedoutby'),
'checkedinby': checkout_dict.get('checkedinby'),
'isactive': True,
'createddate': checkout_dict.get('createddate', datetime.utcnow()),
})
checkout_count += 1
logger.info(f"Migrated {checkout_count} checkout records")
except Exception as e:
logger.warning(f"Could not migrate USB checkouts: {e}")
if dry_run:
logger.info("Dry run - rolling back changes")
target_session.rollback()
else:
target_session.commit()
logger.info("USB migration complete")
finally:
source_session.close()
target_session.close()
def main():
parser = argparse.ArgumentParser(description='Migrate USB devices and checkouts')
parser.add_argument('--source', required=True, help='Source database connection string')
parser.add_argument('--target', help='Target database connection string')
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
args = parser.parse_args()
target = args.target
if not target:
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
from shopdb import create_app
app = create_app()
target = app.config['SQLALCHEMY_DATABASE_URI']
run_migration(args.source, target, args.dry_run)
if __name__ == '__main__':
main()

139
scripts/migration/run_migration.py
View File

@@ -1,139 +0,0 @@
"""
Migration orchestrator - runs all migration steps in order.
This script coordinates the full migration from VBScript ShopDB to Flask.
Usage:
python -m scripts.migration.run_migration --source <connection_string>
"""
import argparse
import logging
import sys
from datetime import datetime
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
def run_full_migration(source_conn_str, target_conn_str, dry_run=False, steps=None):
"""
Run the full migration process.
Args:
source_conn_str: Source database connection string
target_conn_str: Target database connection string
dry_run: If True, don't commit changes
steps: List of specific steps to run, or None for all
"""
from . import migrate_assets
from . import migrate_communications
from . import migrate_notifications
from . import migrate_usb
from . import verify_migration
all_steps = [
('assets', 'Migrate machines to assets', migrate_assets.run_migration),
('communications', 'Update communications FKs', migrate_communications.run_migration),
('notifications', 'Migrate notifications', migrate_notifications.run_migration),
('usb', 'Migrate USB devices', migrate_usb.run_migration),
]
logger.info("=" * 60)
logger.info("SHOPDB MIGRATION")
logger.info(f"Started: {datetime.utcnow().isoformat()}")
logger.info(f"Dry Run: {dry_run}")
logger.info("=" * 60)
results = {}
for step_name, description, migration_func in all_steps:
if steps and step_name not in steps:
logger.info(f"\nSkipping: {description}")
continue
logger.info(f"\n{'=' * 40}")
logger.info(f"Step: {description}")
logger.info('=' * 40)
try:
# Different migrations have different signatures
if step_name == 'communications':
migration_func(target_conn_str, dry_run)
else:
migration_func(source_conn_str, target_conn_str, dry_run)
results[step_name] = 'SUCCESS'
logger.info(f"Step completed: {step_name}")
except Exception as e:
results[step_name] = f'FAILED: {e}'
logger.error(f"Step failed: {step_name} - {e}")
# Ask to continue
if not dry_run:
response = input("Continue with next step? (y/n): ")
if response.lower() != 'y':
logger.info("Migration aborted by user")
break
# Run verification
logger.info(f"\n{'=' * 40}")
logger.info("Running verification...")
logger.info('=' * 40)
try:
verify_migration.run_verification(source_conn_str, target_conn_str)
results['verification'] = 'SUCCESS'
except Exception as e:
results['verification'] = f'FAILED: {e}'
logger.error(f"Verification failed: {e}")
# Summary
logger.info("\n" + "=" * 60)
logger.info("MIGRATION SUMMARY")
logger.info("=" * 60)
for step, result in results.items():
status = "OK" if result == 'SUCCESS' else "FAILED"
logger.info(f" {step}: {status}")
if result != 'SUCCESS':
logger.info(f" {result}")
logger.info("=" * 60)
logger.info(f"Completed: {datetime.utcnow().isoformat()}")
return results
def main():
parser = argparse.ArgumentParser(description='Run full ShopDB migration')
parser.add_argument('--source', required=True, help='Source database connection string')
parser.add_argument('--target', help='Target database connection string')
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
parser.add_argument('--steps', nargs='+',
choices=['assets', 'communications', 'notifications', 'usb'],
help='Specific steps to run')
args = parser.parse_args()
target = args.target
if not target:
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
from shopdb import create_app
app = create_app()
target = app.config['SQLALCHEMY_DATABASE_URI']
results = run_full_migration(args.source, target, args.dry_run, args.steps)
# Exit with error if any step failed
if any(r != 'SUCCESS' for r in results.values()):
sys.exit(1)
if __name__ == '__main__':
main()

174
scripts/migration/verify_migration.py
View File

@@ -1,174 +0,0 @@
"""
Verify data migration integrity.
This script compares record counts between source and target databases
and performs spot checks on data integrity.
Usage:
python -m scripts.migration.verify_migration --source <connection_string>
"""
import argparse
import logging
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def verify_counts(source_session, target_session):
"""Compare record counts between source and target."""
results = {}
# Define table mappings (source -> target)
table_mappings = [
('machines', 'assets', 'Machine to Asset'),
('communications', 'communications', 'Communications'),
('vendors', 'vendors', 'Vendors'),
('locations', 'locations', 'Locations'),
('businessunits', 'businessunits', 'Business Units'),
]
for source_table, target_table, description in table_mappings:
try:
source_count = source_session.execute(text(f"SELECT COUNT(*) FROM {source_table}")).scalar()
except Exception as e:
source_count = f"Error: {e}"
try:
target_count = target_session.execute(text(f"SELECT COUNT(*) FROM {target_table}")).scalar()
except Exception as e:
target_count = f"Error: {e}"
match = source_count == target_count if isinstance(source_count, int) and isinstance(target_count, int) else False
results[description] = {
'source': source_count,
'target': target_count,
'match': match
}
return results
def verify_sample_records(source_session, target_session, sample_size=10):
"""Spot-check sample records for data integrity."""
issues = []
# Sample machine -> asset migration
try:
sample_machines = source_session.execute(text(f"""
SELECT machineid, machinenumber, serialnumber, alias
FROM machines
ORDER BY RAND()
LIMIT {sample_size}
"""))
for machine in sample_machines:
machine_dict = dict(machine._mapping)
# Check if asset exists with same ID
asset = target_session.execute(text("""
SELECT assetid, assetnumber, serialnumber, name
FROM assets
WHERE assetid = :assetid
"""), {'assetid': machine_dict['machineid']}).fetchone()
if not asset:
issues.append(f"Machine {machine_dict['machineid']} not found in assets")
continue
asset_dict = dict(asset._mapping)
# Verify data matches
if machine_dict['machinenumber'] != asset_dict['assetnumber']:
issues.append(f"Asset {asset_dict['assetid']}: machinenumber mismatch")
if machine_dict.get('serialnumber') != asset_dict.get('serialnumber'):
issues.append(f"Asset {asset_dict['assetid']}: serialnumber mismatch")
except Exception as e:
issues.append(f"Could not verify machines: {e}")
return issues
def run_verification(source_conn_str, target_conn_str):
"""
Run migration verification.
Args:
source_conn_str: Source database connection string
target_conn_str: Target database connection string
"""
source_engine = create_engine(source_conn_str)
target_engine = create_engine(target_conn_str)
SourceSession = sessionmaker(bind=source_engine)
TargetSession = sessionmaker(bind=target_engine)
source_session = SourceSession()
target_session = TargetSession()
try:
logger.info("=" * 60)
logger.info("MIGRATION VERIFICATION REPORT")
logger.info("=" * 60)
# Verify counts
logger.info("\nRecord Count Comparison:")
logger.info("-" * 40)
counts = verify_counts(source_session, target_session)
all_match = True
for table, result in counts.items():
status = "OK" if result['match'] else "MISMATCH"
if not result['match']:
all_match = False
logger.info(f" {table}: Source={result['source']}, Target={result['target']} [{status}]")
# Verify sample records
logger.info("\nSample Record Verification:")
logger.info("-" * 40)
issues = verify_sample_records(source_session, target_session)
if issues:
for issue in issues:
logger.warning(f" ! {issue}")
else:
logger.info(" All sample records verified OK")
# Summary
logger.info("\n" + "=" * 60)
if all_match and not issues:
logger.info("VERIFICATION PASSED - Migration looks good!")
else:
logger.warning("VERIFICATION FOUND ISSUES - Review above")
logger.info("=" * 60)
finally:
source_session.close()
target_session.close()
def main():
parser = argparse.ArgumentParser(description='Verify migration integrity')
parser.add_argument('--source', required=True, help='Source database connection string')
parser.add_argument('--target', help='Target database connection string')
args = parser.parse_args()
target = args.target
if not target:
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
from shopdb import create_app
app = create_app()
target = app.config['SQLALCHEMY_DATABASE_URI']
run_verification(args.source, target)
if __name__ == '__main__':
main()