Retire stale legacy-import mappers; add adoption playbook
Cleanup after the reference loader (scripts/site_imports/wjf/) proved out. Removed the drifted direct-SQL migrators - migrate_assets/communications/ notifications/usb.py, run_migration.py, verify_migration.py, and scripts/import_from_mysql.py. They targeted a nonexistent equipment table, the retired Machine model, and columns that no longer exist; nothing imported them. scripts/migration/README.md now points at the import API + the site loader. Kept the one-time SQL fixups (fix_legacy_schema.sql, one-offs/). Added docs/IMPORT-ADOPTION.md: the two-layer import story (stable IMPORT-API contract + per-site loader), stage-ordering + crosswalk guidance, the agent-assisted mapping path, and what the WJ loader demonstrates. Updated the loader README to complete status (all 15 stages, final counts). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
cproudlock
@@ -1,581 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import data from legacy MySQL ShopDB to new Flask ShopDB.
|
||||
|
||||
Usage:
|
||||
cd /home/camp/projects/shopdb-flask
|
||||
source venv/bin/activate
|
||||
python scripts/import_from_mysql.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import pymysql
|
||||
from datetime import datetime
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.env'))
|
||||
|
||||
# MySQL connection settings
|
||||
MYSQL_CONFIG = {
|
||||
'host': '127.0.0.1',
|
||||
'port': 3306,
|
||||
'user': 'root',
|
||||
'password': 'rootpassword',
|
||||
'database': 'shopdb',
|
||||
'charset': 'utf8mb4',
|
||||
'cursorclass': pymysql.cursors.DictCursor
|
||||
}
|
||||
|
||||
|
||||
def get_mysql_connection():
|
||||
"""Get MySQL connection."""
|
||||
return pymysql.connect(**MYSQL_CONFIG)
|
||||
|
||||
|
||||
def import_vendors(mysql_conn, db, Vendor):
|
||||
"""Import vendors from MySQL."""
|
||||
print("Importing vendors...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM vendors WHERE isactive = 1")
|
||||
vendors = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for v in vendors:
|
||||
existing = Vendor.query.filter_by(vendor=v['vendor']).first()
|
||||
if not existing:
|
||||
vendor = Vendor(
|
||||
vendor=v['vendor'],
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(vendor)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} vendors")
|
||||
return count
|
||||
|
||||
|
||||
def import_machinetypes(mysql_conn, db, MachineType):
|
||||
"""Import machine types from MySQL with category mapping."""
|
||||
print("Importing machine types...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM machinetypes WHERE isactive = 1")
|
||||
types = cursor.fetchall()
|
||||
|
||||
# Category mapping based on machinetype name
|
||||
pc_types = ['PC']
|
||||
network_types = ['Access Point', 'IDF', 'Switch', 'Server', 'Camera']
|
||||
printer_types = ['Printer']
|
||||
|
||||
count = 0
|
||||
for t in types:
|
||||
existing = MachineType.query.filter_by(machinetype=t['machinetype']).first()
|
||||
if not existing:
|
||||
# Determine category
|
||||
if t['machinetype'] in pc_types:
|
||||
category = 'PC'
|
||||
elif t['machinetype'] in network_types:
|
||||
category = 'Network'
|
||||
elif t['machinetype'] in printer_types:
|
||||
category = 'Printer'
|
||||
else:
|
||||
category = 'Equipment'
|
||||
|
||||
mt = MachineType(
|
||||
machinetype=t['machinetype'],
|
||||
category=category,
|
||||
description=t.get('machinedescription'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(mt)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} machine types")
|
||||
return count
|
||||
|
||||
|
||||
def import_pctypes(mysql_conn, db, PCType):
|
||||
"""Import PC types from MySQL."""
|
||||
print("Importing PC types...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM pctype WHERE isactive = '1'")
|
||||
types = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for t in types:
|
||||
existing = PCType.query.filter_by(pctype=t['typename']).first()
|
||||
if not existing:
|
||||
pctype = PCType(
|
||||
pctype=t['typename'],
|
||||
description=t.get('description'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(pctype)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} PC types")
|
||||
return count
|
||||
|
||||
|
||||
def import_businessunits(mysql_conn, db, BusinessUnit):
|
||||
"""Import business units from MySQL."""
|
||||
print("Importing business units...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM businessunits WHERE isactive = 1")
|
||||
units = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for bu in units:
|
||||
existing = BusinessUnit.query.filter_by(businessunit=bu['businessunit']).first()
|
||||
if not existing:
|
||||
unit = BusinessUnit(
|
||||
businessunit=bu['businessunit'],
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(unit)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} business units")
|
||||
return count
|
||||
|
||||
|
||||
def import_statuses(mysql_conn, db, MachineStatus):
|
||||
"""Import machine statuses from MySQL."""
|
||||
print("Importing machine statuses...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM machinestatus WHERE isactive = 1")
|
||||
statuses = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for s in statuses:
|
||||
existing = MachineStatus.query.filter_by(status=s['machinestatus']).first()
|
||||
if not existing:
|
||||
status = MachineStatus(
|
||||
status=s['machinestatus'],
|
||||
description=s.get('statusdescription'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(status)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} statuses")
|
||||
return count
|
||||
|
||||
|
||||
def import_operatingsystems(mysql_conn, db, OperatingSystem):
|
||||
"""Import operating systems from MySQL."""
|
||||
print("Importing operating systems...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM operatingsystems")
|
||||
os_list = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for os_item in os_list:
|
||||
os_name = os_item.get('operatingsystem') or os_item.get('osname')
|
||||
if not os_name:
|
||||
continue
|
||||
|
||||
existing = OperatingSystem.query.filter_by(osname=os_name).first()
|
||||
if not existing:
|
||||
os_obj = OperatingSystem(
|
||||
osname=os_name,
|
||||
osversion=os_item.get('osversion'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(os_obj)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} operating systems")
|
||||
return count
|
||||
|
||||
|
||||
def import_models(mysql_conn, db, Model, Vendor, MachineType):
|
||||
"""Import models from MySQL."""
|
||||
print("Importing models...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT m.*, v.vendor as vendor_name, mt.machinetype as type_name
|
||||
FROM models m
|
||||
LEFT JOIN vendors v ON m.vendorid = v.vendorid
|
||||
LEFT JOIN machinetypes mt ON m.machinetypeid = mt.machinetypeid
|
||||
WHERE m.isactive = 1
|
||||
""")
|
||||
models = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for m in models:
|
||||
existing = Model.query.filter_by(modelnumber=m['modelnumber']).first()
|
||||
if not existing:
|
||||
# Find vendor and machinetype in new db
|
||||
vendor = Vendor.query.filter_by(vendor=m['vendor_name']).first() if m['vendor_name'] else None
|
||||
machinetype = MachineType.query.filter_by(machinetype=m['type_name']).first() if m['type_name'] else None
|
||||
|
||||
model = Model(
|
||||
modelnumber=m['modelnumber'],
|
||||
vendorid=vendor.vendorid if vendor else None,
|
||||
machinetypeid=machinetype.machinetypeid if machinetype else None,
|
||||
notes=m.get('notes'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(model)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} models")
|
||||
return count
|
||||
|
||||
|
||||
def import_relationshiptypes(mysql_conn, db, RelationshipType):
|
||||
"""Import relationship types from MySQL."""
|
||||
print("Importing relationship/connection types...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("SELECT * FROM relationshiptypes WHERE isactive = 1")
|
||||
types = cursor.fetchall()
|
||||
|
||||
count = 0
|
||||
for rt in types:
|
||||
existing = RelationshipType.query.filter_by(relationshiptype=rt['relationshiptype']).first()
|
||||
if not existing:
|
||||
rel_type = RelationshipType(
|
||||
relationshiptype=rt['relationshiptype'],
|
||||
description=rt.get('description'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(rel_type)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} relationship types")
|
||||
return count
|
||||
|
||||
|
||||
def import_machines(mysql_conn, db, Machine, MachineType, MachineStatus,
|
||||
Vendor, Model, BusinessUnit, OperatingSystem, Location,
|
||||
Communication, CommunicationType, PCType):
|
||||
"""Import machines (Equipment and PCs) from MySQL."""
|
||||
print("Importing machines...")
|
||||
cursor = mysql_conn.cursor()
|
||||
|
||||
# Get machines with related data
|
||||
cursor.execute("""
|
||||
SELECT m.*,
|
||||
mt.machinetype as type_name,
|
||||
ms.machinestatus as status_name,
|
||||
v.vendor as vendor_name,
|
||||
mdl.modelnumber as model_name,
|
||||
bu.businessunit as bu_name,
|
||||
os.operatingsystem as os_name,
|
||||
pt.typename as pctype_name
|
||||
FROM machines m
|
||||
LEFT JOIN machinetypes mt ON m.machinetypeid = mt.machinetypeid
|
||||
LEFT JOIN machinestatus ms ON m.machinestatusid = ms.machinestatusid
|
||||
LEFT JOIN models mdl ON m.modelnumberid = mdl.modelnumberid
|
||||
LEFT JOIN vendors v ON mdl.vendorid = v.vendorid
|
||||
LEFT JOIN businessunits bu ON m.businessunitid = bu.businessunitid
|
||||
LEFT JOIN operatingsystems os ON m.osid = os.osid
|
||||
LEFT JOIN pctype pt ON m.pctypeid = pt.pctypeid
|
||||
WHERE m.isactive = 1
|
||||
""")
|
||||
machines = cursor.fetchall()
|
||||
|
||||
# Get or create IP communication type
|
||||
ip_comtype = CommunicationType.query.filter_by(comtype='IP').first()
|
||||
if not ip_comtype:
|
||||
ip_comtype = CommunicationType(comtype='IP', description='IP Network')
|
||||
db.session.add(ip_comtype)
|
||||
db.session.flush()
|
||||
|
||||
# Build lookup maps
|
||||
type_map = {t.machinetype: t for t in MachineType.query.all()}
|
||||
status_map = {s.status: s for s in MachineStatus.query.all()}
|
||||
vendor_map = {v.vendor: v for v in Vendor.query.all()}
|
||||
model_map = {m.modelnumber: m for m in Model.query.all()}
|
||||
bu_map = {b.businessunit: b for b in BusinessUnit.query.all()}
|
||||
os_map = {o.osname: o for o in OperatingSystem.query.all()}
|
||||
pctype_map = {p.pctype: p for p in PCType.query.all()}
|
||||
|
||||
# Track old->new ID mapping for relationships
|
||||
machine_id_map = {}
|
||||
|
||||
count = 0
|
||||
comm_count = 0
|
||||
skipped = 0
|
||||
for m in machines:
|
||||
# Skip machines without a machinenumber
|
||||
if not m.get('machinenumber'):
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Check if already exists
|
||||
existing = Machine.query.filter_by(machinenumber=m['machinenumber']).first()
|
||||
if existing:
|
||||
machine_id_map[m['machineid']] = existing.machineid
|
||||
continue
|
||||
|
||||
# Get related objects
|
||||
machinetype = type_map.get(m['type_name'])
|
||||
status = status_map.get(m['status_name'])
|
||||
vendor = vendor_map.get(m['vendor_name'])
|
||||
model = model_map.get(m['model_name'])
|
||||
bu = bu_map.get(m['bu_name'])
|
||||
os_obj = os_map.get(m['os_name'])
|
||||
pctype = pctype_map.get(m['pctype_name'])
|
||||
|
||||
machine = Machine(
|
||||
machinenumber=m['machinenumber'],
|
||||
alias=m.get('alias'),
|
||||
hostname=m.get('hostname'),
|
||||
serialnumber=m.get('serialnumber'),
|
||||
machinetypeid=machinetype.machinetypeid if machinetype else None,
|
||||
pctypeid=pctype.pctypeid if pctype else None,
|
||||
statusid=status.statusid if status else None,
|
||||
vendorid=vendor.vendorid if vendor else None,
|
||||
modelnumberid=model.modelnumberid if model else None,
|
||||
businessunitid=bu.businessunitid if bu else None,
|
||||
osid=os_obj.osid if os_obj else None,
|
||||
mapleft=m.get('mapleft'),
|
||||
maptop=m.get('maptop'),
|
||||
isvnc=bool(m.get('isvnc')),
|
||||
iswinrm=bool(m.get('iswinrm')),
|
||||
islocationonly=bool(m.get('islocationonly')),
|
||||
loggedinuser=m.get('loggedinuser'),
|
||||
notes=m.get('machinenotes'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(machine)
|
||||
db.session.flush() # Get the new ID
|
||||
|
||||
machine_id_map[m['machineid']] = machine.machineid
|
||||
count += 1
|
||||
|
||||
# Import IP addresses
|
||||
if m.get('ipaddress1'):
|
||||
comm = Communication(
|
||||
machineid=machine.machineid,
|
||||
comtypeid=ip_comtype.comtypeid,
|
||||
ipaddress=m['ipaddress1'],
|
||||
isprimary=True
|
||||
)
|
||||
db.session.add(comm)
|
||||
comm_count += 1
|
||||
|
||||
if m.get('ipaddress2'):
|
||||
comm = Communication(
|
||||
machineid=machine.machineid,
|
||||
comtypeid=ip_comtype.comtypeid,
|
||||
ipaddress=m['ipaddress2'],
|
||||
isprimary=False
|
||||
)
|
||||
db.session.add(comm)
|
||||
comm_count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} machines with {comm_count} IP addresses (skipped {skipped} invalid)")
|
||||
return machine_id_map
|
||||
|
||||
|
||||
def import_relationships(mysql_conn, db, MachineRelationship, RelationshipType, machine_id_map):
|
||||
"""Import machine relationships from MySQL."""
|
||||
print("Importing machine relationships...")
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT mr.*, rt.relationshiptype
|
||||
FROM machinerelationships mr
|
||||
JOIN relationshiptypes rt ON mr.relationshiptypeid = rt.relationshiptypeid
|
||||
WHERE mr.isactive = 1
|
||||
""")
|
||||
relationships = cursor.fetchall()
|
||||
|
||||
# Build relationship type map
|
||||
type_map = {t.relationshiptype: t for t in RelationshipType.query.all()}
|
||||
|
||||
count = 0
|
||||
skipped = 0
|
||||
for r in relationships:
|
||||
# Map old IDs to new IDs
|
||||
parent_id = machine_id_map.get(r['machineid'])
|
||||
child_id = machine_id_map.get(r['related_machineid'])
|
||||
|
||||
if not parent_id or not child_id:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
rel_type = type_map.get(r['relationshiptype'])
|
||||
if not rel_type:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Check if already exists
|
||||
existing = MachineRelationship.query.filter_by(
|
||||
parentmachineid=parent_id,
|
||||
childmachineid=child_id,
|
||||
relationshiptypeid=rel_type.relationshiptypeid
|
||||
).first()
|
||||
|
||||
if not existing:
|
||||
relationship = MachineRelationship(
|
||||
parentmachineid=parent_id,
|
||||
childmachineid=child_id,
|
||||
relationshiptypeid=rel_type.relationshiptypeid,
|
||||
notes=r.get('relationship_notes')
|
||||
)
|
||||
db.session.add(relationship)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} relationships (skipped {skipped})")
|
||||
return count
|
||||
|
||||
|
||||
def import_printers(mysql_conn, db, Machine, MachineType, Model, Vendor,
|
||||
Communication, CommunicationType):
|
||||
"""Import printers from MySQL."""
|
||||
print("Importing printers...")
|
||||
|
||||
# First, ensure we have a Printer machine type
|
||||
printer_type = MachineType.query.filter_by(machinetype='Printer').first()
|
||||
if not printer_type:
|
||||
printer_type = MachineType(machinetype='Printer', category='Printer')
|
||||
db.session.add(printer_type)
|
||||
db.session.flush()
|
||||
|
||||
# Get or create IP communication type
|
||||
ip_comtype = CommunicationType.query.filter_by(comtype='IP').first()
|
||||
if not ip_comtype:
|
||||
ip_comtype = CommunicationType(comtype='IP', description='IP Network')
|
||||
db.session.add(ip_comtype)
|
||||
db.session.flush()
|
||||
|
||||
cursor = mysql_conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT p.*, m.modelnumber, v.vendor as vendor_name
|
||||
FROM printers p
|
||||
LEFT JOIN models m ON p.modelid = m.modelnumberid
|
||||
LEFT JOIN vendors v ON m.vendorid = v.vendorid
|
||||
WHERE p.isactive = 1
|
||||
""")
|
||||
printers = cursor.fetchall()
|
||||
|
||||
# Build lookup maps
|
||||
model_map = {m.modelnumber: m for m in Model.query.all()}
|
||||
vendor_map = {v.vendor: v for v in Vendor.query.all()}
|
||||
|
||||
count = 0
|
||||
comm_count = 0
|
||||
for p in printers:
|
||||
# Use windows name as machine number
|
||||
machine_number = p.get('printerwindowsname') or f"Printer_{p['printerid']}"
|
||||
|
||||
existing = Machine.query.filter_by(machinenumber=machine_number).first()
|
||||
if existing:
|
||||
continue
|
||||
|
||||
model = model_map.get(p.get('modelnumber'))
|
||||
vendor = vendor_map.get(p.get('vendor_name'))
|
||||
|
||||
machine = Machine(
|
||||
machinenumber=machine_number,
|
||||
alias=p.get('printercsfname'),
|
||||
hostname=p.get('fqdn'),
|
||||
serialnumber=p.get('serialnumber'),
|
||||
machinetypeid=printer_type.machinetypeid,
|
||||
vendorid=vendor.vendorid if vendor else None,
|
||||
modelnumberid=model.modelnumberid if model else None,
|
||||
mapleft=p.get('mapleft'),
|
||||
maptop=p.get('maptop'),
|
||||
notes=p.get('printernotes'),
|
||||
isactive=True
|
||||
)
|
||||
db.session.add(machine)
|
||||
db.session.flush()
|
||||
count += 1
|
||||
|
||||
# Import IP address
|
||||
if p.get('ipaddress'):
|
||||
comm = Communication(
|
||||
machineid=machine.machineid,
|
||||
comtypeid=ip_comtype.comtypeid,
|
||||
ipaddress=p['ipaddress'],
|
||||
isprimary=True
|
||||
)
|
||||
db.session.add(comm)
|
||||
comm_count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" Imported {count} printers with {comm_count} IP addresses")
|
||||
return count
|
||||
|
||||
|
||||
def main():
|
||||
"""Main import function."""
|
||||
print("=" * 60)
|
||||
print("ShopDB MySQL to Flask Migration")
|
||||
print("=" * 60)
|
||||
|
||||
# Initialize Flask app
|
||||
from shopdb import create_app
|
||||
from shopdb.extensions import db
|
||||
from shopdb.core.models import (
|
||||
Machine, MachineType, MachineStatus, Vendor, Model,
|
||||
BusinessUnit, OperatingSystem, Location, PCType
|
||||
)
|
||||
from shopdb.core.models.communication import Communication, CommunicationType
|
||||
from shopdb.core.models.relationship import MachineRelationship, RelationshipType
|
||||
|
||||
app = create_app()
|
||||
|
||||
with app.app_context():
|
||||
# Connect to MySQL
|
||||
print("\nConnecting to MySQL...")
|
||||
mysql_conn = get_mysql_connection()
|
||||
print(" Connected!")
|
||||
|
||||
try:
|
||||
# Import reference data
|
||||
print("\n--- Reference Data ---")
|
||||
import_vendors(mysql_conn, db, Vendor)
|
||||
import_machinetypes(mysql_conn, db, MachineType)
|
||||
import_pctypes(mysql_conn, db, PCType)
|
||||
import_businessunits(mysql_conn, db, BusinessUnit)
|
||||
import_statuses(mysql_conn, db, MachineStatus)
|
||||
import_operatingsystems(mysql_conn, db, OperatingSystem)
|
||||
import_models(mysql_conn, db, Model, Vendor, MachineType)
|
||||
import_relationshiptypes(mysql_conn, db, RelationshipType)
|
||||
|
||||
# Import machines
|
||||
print("\n--- Machines ---")
|
||||
machine_id_map = import_machines(
|
||||
mysql_conn, db, Machine, MachineType, MachineStatus,
|
||||
Vendor, Model, BusinessUnit, OperatingSystem, Location,
|
||||
Communication, CommunicationType, PCType
|
||||
)
|
||||
|
||||
# Import relationships
|
||||
print("\n--- Relationships ---")
|
||||
import_relationships(mysql_conn, db, MachineRelationship, RelationshipType, machine_id_map)
|
||||
|
||||
# Import printers
|
||||
print("\n--- Printers ---")
|
||||
import_printers(mysql_conn, db, Machine, MachineType, Model, Vendor,
|
||||
Communication, CommunicationType)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Import complete!")
|
||||
print("=" * 60)
|
||||
|
||||
finally:
|
||||
mysql_conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,285 +0,0 @@
|
||||
"""
|
||||
Migrate machines table to assets + extension tables.
|
||||
|
||||
This script migrates data from the legacy machines table to the new
|
||||
Asset architecture with plugin-owned extension tables.
|
||||
|
||||
Strategy:
|
||||
1. Preserve IDs: assets.assetid = original machines.machineid
|
||||
2. Create asset record, then type-specific extension record
|
||||
3. Map machine types to asset types:
|
||||
- MachineType = Equipment -> equipment extension
|
||||
- MachineType = PC -> computers extension
|
||||
- MachineType = Network/Camera/etc -> network_devices extension
|
||||
- Printers -> handled separately by printers plugin
|
||||
|
||||
Usage:
|
||||
python -m scripts.migration.migrate_assets --source <connection_string>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_machine_type_mapping():
|
||||
"""Map legacy machine type IDs to asset types."""
|
||||
return {
|
||||
# Equipment types
|
||||
'CNC': 'equipment',
|
||||
'CMM': 'equipment',
|
||||
'Lathe': 'equipment',
|
||||
'Grinder': 'equipment',
|
||||
'EDM': 'equipment',
|
||||
'Mill': 'equipment',
|
||||
'Press': 'equipment',
|
||||
'Robot': 'equipment',
|
||||
'Part Marker': 'equipment',
|
||||
# PC types
|
||||
'PC': 'computer',
|
||||
'Workstation': 'computer',
|
||||
'Laptop': 'computer',
|
||||
'Server': 'computer',
|
||||
# Network types
|
||||
'Switch': 'network_device',
|
||||
'Router': 'network_device',
|
||||
'Access Point': 'network_device',
|
||||
'Camera': 'network_device',
|
||||
'IDF': 'network_device',
|
||||
'MDF': 'network_device',
|
||||
'Firewall': 'network_device',
|
||||
}
|
||||
|
||||
|
||||
def migrate_machine_to_asset(machine_row, asset_type_id, target_session):
|
||||
"""
|
||||
Create an Asset record from a Machine record.
|
||||
|
||||
Args:
|
||||
machine_row: Row from source machines table
|
||||
asset_type_id: Target asset type ID
|
||||
target_session: SQLAlchemy session for target database
|
||||
|
||||
Returns:
|
||||
Created asset ID
|
||||
"""
|
||||
# Insert into assets table
|
||||
target_session.execute(text("""
|
||||
INSERT INTO assets (
|
||||
assetid, assetnumber, name, serialnumber,
|
||||
assettypeid, statusid, locationid, businessunitid,
|
||||
mapx, mapy, notes, isactive, createddate, modifieddate
|
||||
) VALUES (
|
||||
:assetid, :assetnumber, :name, :serialnumber,
|
||||
:assettypeid, :statusid, :locationid, :businessunitid,
|
||||
:mapx, :mapy, :notes, :isactive, :createddate, :modifieddate
|
||||
)
|
||||
"""), {
|
||||
'assetid': machine_row['machineid'],
|
||||
'assetnumber': machine_row['machinenumber'],
|
||||
'name': machine_row.get('alias'),
|
||||
'serialnumber': machine_row.get('serialnumber'),
|
||||
'assettypeid': asset_type_id,
|
||||
'statusid': machine_row.get('statusid', 1),
|
||||
'locationid': machine_row.get('locationid'),
|
||||
'businessunitid': machine_row.get('businessunitid'),
|
||||
'mapx': machine_row.get('mapx'),
|
||||
'mapy': machine_row.get('mapy'),
|
||||
'notes': machine_row.get('notes'),
|
||||
'isactive': machine_row.get('isactive', True),
|
||||
'createddate': machine_row.get('createddate', datetime.utcnow()),
|
||||
'modifieddate': machine_row.get('modifieddate', datetime.utcnow()),
|
||||
})
|
||||
|
||||
return machine_row['machineid']
|
||||
|
||||
|
||||
def migrate_equipment(machine_row, asset_id, target_session):
|
||||
"""Create equipment extension record."""
|
||||
target_session.execute(text("""
|
||||
INSERT INTO equipment (
|
||||
assetid, equipmenttypeid, vendorid, modelnumberid,
|
||||
requiresmanualconfig, islocationonly, isactive, createddate
|
||||
) VALUES (
|
||||
:assetid, :equipmenttypeid, :vendorid, :modelnumberid,
|
||||
:requiresmanualconfig, :islocationonly, :isactive, :createddate
|
||||
)
|
||||
"""), {
|
||||
'assetid': asset_id,
|
||||
'equipmenttypeid': machine_row.get('machinetypeid'), # May need mapping
|
||||
'vendorid': machine_row.get('vendorid'),
|
||||
'modelnumberid': machine_row.get('modelnumberid'),
|
||||
'requiresmanualconfig': machine_row.get('requiresmanualconfig', False),
|
||||
'islocationonly': machine_row.get('islocationonly', False),
|
||||
'isactive': True,
|
||||
'createddate': datetime.utcnow(),
|
||||
})
|
||||
|
||||
|
||||
def migrate_computer(machine_row, asset_id, target_session):
|
||||
"""Create computer extension record."""
|
||||
target_session.execute(text("""
|
||||
INSERT INTO computers (
|
||||
assetid, computertypeid, vendorid, operatingsystemid,
|
||||
hostname, currentuserid, lastuserid, lastboottime,
|
||||
lastzabbixsync, isvnc, isactive, createddate
|
||||
) VALUES (
|
||||
:assetid, :computertypeid, :vendorid, :operatingsystemid,
|
||||
:hostname, :currentuserid, :lastuserid, :lastboottime,
|
||||
:lastzabbixsync, :isvnc, :isactive, :createddate
|
||||
)
|
||||
"""), {
|
||||
'assetid': asset_id,
|
||||
'computertypeid': machine_row.get('pctypeid'),
|
||||
'vendorid': machine_row.get('vendorid'),
|
||||
'operatingsystemid': machine_row.get('operatingsystemid'),
|
||||
'hostname': machine_row.get('hostname'),
|
||||
'currentuserid': machine_row.get('currentuserid'),
|
||||
'lastuserid': machine_row.get('lastuserid'),
|
||||
'lastboottime': machine_row.get('lastboottime'),
|
||||
'lastzabbixsync': machine_row.get('lastzabbixsync'),
|
||||
'isvnc': machine_row.get('isvnc', False),
|
||||
'isactive': True,
|
||||
'createddate': datetime.utcnow(),
|
||||
})
|
||||
|
||||
|
||||
def migrate_network_device(machine_row, asset_id, target_session):
|
||||
"""Create network device extension record."""
|
||||
target_session.execute(text("""
|
||||
INSERT INTO networkdevices (
|
||||
assetid, networkdevicetypeid, vendorid, hostname,
|
||||
firmwareversion, portcount, ispoe, ismanaged,
|
||||
isactive, createddate
|
||||
) VALUES (
|
||||
:assetid, :networkdevicetypeid, :vendorid, :hostname,
|
||||
:firmwareversion, :portcount, :ispoe, :ismanaged,
|
||||
:isactive, :createddate
|
||||
)
|
||||
"""), {
|
||||
'assetid': asset_id,
|
||||
'networkdevicetypeid': machine_row.get('machinetypeid'), # May need mapping
|
||||
'vendorid': machine_row.get('vendorid'),
|
||||
'hostname': machine_row.get('hostname'),
|
||||
'firmwareversion': machine_row.get('firmwareversion'),
|
||||
'portcount': machine_row.get('portcount'),
|
||||
'ispoe': machine_row.get('ispoe', False),
|
||||
'ismanaged': machine_row.get('ismanaged', False),
|
||||
'isactive': True,
|
||||
'createddate': datetime.utcnow(),
|
||||
})
|
||||
|
||||
|
||||
def run_migration(source_conn_str, target_conn_str, dry_run=False):
|
||||
"""
|
||||
Run the full migration from machines to assets.
|
||||
|
||||
Args:
|
||||
source_conn_str: Connection string for source (VBScript) database
|
||||
target_conn_str: Connection string for target (Flask) database
|
||||
dry_run: If True, don't commit changes
|
||||
"""
|
||||
source_engine = create_engine(source_conn_str)
|
||||
target_engine = create_engine(target_conn_str)
|
||||
|
||||
SourceSession = sessionmaker(bind=source_engine)
|
||||
TargetSession = sessionmaker(bind=target_engine)
|
||||
|
||||
source_session = SourceSession()
|
||||
target_session = TargetSession()
|
||||
|
||||
try:
|
||||
# Get asset type mappings from target database
|
||||
asset_types = {}
|
||||
result = target_session.execute(text("SELECT assettypeid, assettype FROM assettypes"))
|
||||
for row in result:
|
||||
asset_types[row.assettype] = row.assettypeid
|
||||
|
||||
# Get machine type to asset type mapping
|
||||
type_mapping = get_machine_type_mapping()
|
||||
|
||||
# Fetch all machines from source
|
||||
machines = source_session.execute(text("""
|
||||
SELECT m.*, mt.machinetype
|
||||
FROM machines m
|
||||
LEFT JOIN machinetypes mt ON m.machinetypeid = mt.machinetypeid
|
||||
"""))
|
||||
|
||||
migrated = 0
|
||||
errors = 0
|
||||
|
||||
for machine in machines:
|
||||
machine_dict = dict(machine._mapping)
|
||||
|
||||
try:
|
||||
# Determine asset type
|
||||
machine_type_name = machine_dict.get('machinetype', '')
|
||||
asset_type_name = type_mapping.get(machine_type_name, 'equipment')
|
||||
asset_type_id = asset_types.get(asset_type_name)
|
||||
|
||||
if not asset_type_id:
|
||||
logger.warning(f"Unknown asset type for machine {machine_dict['machineid']}: {machine_type_name}")
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
# Create asset record
|
||||
asset_id = migrate_machine_to_asset(machine_dict, asset_type_id, target_session)
|
||||
|
||||
# Create extension record based on type
|
||||
if asset_type_name == 'equipment':
|
||||
migrate_equipment(machine_dict, asset_id, target_session)
|
||||
elif asset_type_name == 'computer':
|
||||
migrate_computer(machine_dict, asset_id, target_session)
|
||||
elif asset_type_name == 'network_device':
|
||||
migrate_network_device(machine_dict, asset_id, target_session)
|
||||
|
||||
migrated += 1
|
||||
|
||||
if migrated % 100 == 0:
|
||||
logger.info(f"Migrated {migrated} machines...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error migrating machine {machine_dict.get('machineid')}: {e}")
|
||||
errors += 1
|
||||
|
||||
if dry_run:
|
||||
logger.info("Dry run - rolling back changes")
|
||||
target_session.rollback()
|
||||
else:
|
||||
target_session.commit()
|
||||
|
||||
logger.info(f"Migration complete: {migrated} migrated, {errors} errors")
|
||||
|
||||
finally:
|
||||
source_session.close()
|
||||
target_session.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Migrate machines to assets')
|
||||
parser.add_argument('--source', required=True, help='Source database connection string')
|
||||
parser.add_argument('--target', help='Target database connection string (default: app config)')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target = args.target
|
||||
if not target:
|
||||
# Load from Flask config
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
from shopdb import create_app
|
||||
app = create_app()
|
||||
target = app.config['SQLALCHEMY_DATABASE_URI']
|
||||
|
||||
run_migration(args.source, target, args.dry_run)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,113 +0,0 @@
|
||||
"""
|
||||
Migrate communications table to use assetid instead of machineid.
|
||||
|
||||
This script updates the communications table FK from machineid to assetid.
|
||||
Since assetid matches the original machineid, this is mostly a schema update.
|
||||
|
||||
Usage:
|
||||
python -m scripts.migration.migrate_communications
|
||||
"""
|
||||
|
||||
import logging
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_migration(conn_str, dry_run=False):
|
||||
"""
|
||||
Update communications to use assetid.
|
||||
|
||||
Args:
|
||||
conn_str: Database connection string
|
||||
dry_run: If True, don't commit changes
|
||||
"""
|
||||
engine = create_engine(conn_str)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
try:
|
||||
# Check if assetid column already exists
|
||||
result = session.execute(text("""
|
||||
SELECT COUNT(*) FROM information_schema.columns
|
||||
WHERE table_name = 'communications' AND column_name = 'assetid'
|
||||
"""))
|
||||
has_assetid = result.scalar() > 0
|
||||
|
||||
if not has_assetid:
|
||||
logger.info("Adding assetid column to communications table...")
|
||||
|
||||
# Add assetid column
|
||||
session.execute(text("""
|
||||
ALTER TABLE communications
|
||||
ADD COLUMN assetid INT NULL
|
||||
"""))
|
||||
|
||||
# Copy machineid values to assetid
|
||||
session.execute(text("""
|
||||
UPDATE communications
|
||||
SET assetid = machineid
|
||||
WHERE machineid IS NOT NULL
|
||||
"""))
|
||||
|
||||
# Add FK constraint (optional, depends on DB)
|
||||
try:
|
||||
session.execute(text("""
|
||||
ALTER TABLE communications
|
||||
ADD CONSTRAINT fk_comm_asset
|
||||
FOREIGN KEY (assetid) REFERENCES assets(assetid)
|
||||
"""))
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not add FK constraint: {e}")
|
||||
|
||||
logger.info("assetid column added and populated")
|
||||
else:
|
||||
logger.info("assetid column already exists")
|
||||
|
||||
# Count records
|
||||
result = session.execute(text("""
|
||||
SELECT COUNT(*) FROM communications WHERE assetid IS NOT NULL
|
||||
"""))
|
||||
count = result.scalar()
|
||||
logger.info(f"Communications with assetid: {count}")
|
||||
|
||||
if dry_run:
|
||||
logger.info("Dry run - rolling back changes")
|
||||
session.rollback()
|
||||
else:
|
||||
session.commit()
|
||||
logger.info("Migration complete")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Migration error: {e}")
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
parser = argparse.ArgumentParser(description='Migrate communications to use assetid')
|
||||
parser.add_argument('--connection', help='Database connection string')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
conn_str = args.connection
|
||||
if not conn_str:
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
from shopdb import create_app
|
||||
app = create_app()
|
||||
conn_str = app.config['SQLALCHEMY_DATABASE_URI']
|
||||
|
||||
run_migration(conn_str, args.dry_run)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,139 +0,0 @@
|
||||
"""
|
||||
Migrate notifications from legacy database.
|
||||
|
||||
This script migrates notification data from the VBScript database
|
||||
to the new notifications plugin schema.
|
||||
|
||||
Usage:
|
||||
python -m scripts.migration.migrate_notifications --source <connection_string>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_notification_type_mapping(target_session):
|
||||
"""Get mapping of type names to IDs in target database."""
|
||||
result = target_session.execute(text(
|
||||
"SELECT notificationtypeid, typename FROM notificationtypes"
|
||||
))
|
||||
return {row.typename.lower(): row.notificationtypeid for row in result}
|
||||
|
||||
|
||||
def run_migration(source_conn_str, target_conn_str, dry_run=False):
|
||||
"""
|
||||
Run notification migration.
|
||||
|
||||
Args:
|
||||
source_conn_str: Source database connection string
|
||||
target_conn_str: Target database connection string
|
||||
dry_run: If True, don't commit changes
|
||||
"""
|
||||
source_engine = create_engine(source_conn_str)
|
||||
target_engine = create_engine(target_conn_str)
|
||||
|
||||
SourceSession = sessionmaker(bind=source_engine)
|
||||
TargetSession = sessionmaker(bind=target_engine)
|
||||
|
||||
source_session = SourceSession()
|
||||
target_session = TargetSession()
|
||||
|
||||
try:
|
||||
# Get type mappings
|
||||
type_mapping = get_notification_type_mapping(target_session)
|
||||
|
||||
# Default type if not found
|
||||
default_type_id = type_mapping.get('general', 1)
|
||||
|
||||
# Fetch notifications from source
|
||||
# Adjust column names based on actual legacy schema
|
||||
notifications = source_session.execute(text("""
|
||||
SELECT n.*, nt.typename
|
||||
FROM notifications n
|
||||
LEFT JOIN notificationtypes nt ON n.notificationtypeid = nt.notificationtypeid
|
||||
"""))
|
||||
|
||||
migrated = 0
|
||||
errors = 0
|
||||
|
||||
for notif in notifications:
|
||||
notif_dict = dict(notif._mapping)
|
||||
|
||||
try:
|
||||
# Map notification type
|
||||
type_name = (notif_dict.get('typename') or 'general').lower()
|
||||
type_id = type_mapping.get(type_name, default_type_id)
|
||||
|
||||
# Insert into target
|
||||
target_session.execute(text("""
|
||||
INSERT INTO notifications (
|
||||
title, message, notificationtypeid,
|
||||
startdate, enddate, ispinned, showbanner, allday,
|
||||
linkurl, affectedsystems, isactive, createddate
|
||||
) VALUES (
|
||||
:title, :message, :notificationtypeid,
|
||||
:startdate, :enddate, :ispinned, :showbanner, :allday,
|
||||
:linkurl, :affectedsystems, :isactive, :createddate
|
||||
)
|
||||
"""), {
|
||||
'title': notif_dict.get('title', 'Untitled'),
|
||||
'message': notif_dict.get('message', ''),
|
||||
'notificationtypeid': type_id,
|
||||
'startdate': notif_dict.get('startdate', datetime.utcnow()),
|
||||
'enddate': notif_dict.get('enddate'),
|
||||
'ispinned': notif_dict.get('ispinned', False),
|
||||
'showbanner': notif_dict.get('showbanner', True),
|
||||
'allday': notif_dict.get('allday', True),
|
||||
'linkurl': notif_dict.get('linkurl'),
|
||||
'affectedsystems': notif_dict.get('affectedsystems'),
|
||||
'isactive': notif_dict.get('isactive', True),
|
||||
'createddate': notif_dict.get('createddate', datetime.utcnow()),
|
||||
})
|
||||
|
||||
migrated += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error migrating notification: {e}")
|
||||
errors += 1
|
||||
|
||||
if dry_run:
|
||||
logger.info("Dry run - rolling back changes")
|
||||
target_session.rollback()
|
||||
else:
|
||||
target_session.commit()
|
||||
|
||||
logger.info(f"Migration complete: {migrated} migrated, {errors} errors")
|
||||
|
||||
finally:
|
||||
source_session.close()
|
||||
target_session.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Migrate notifications')
|
||||
parser.add_argument('--source', required=True, help='Source database connection string')
|
||||
parser.add_argument('--target', help='Target database connection string')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target = args.target
|
||||
if not target:
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
from shopdb import create_app
|
||||
app = create_app()
|
||||
target = app.config['SQLALCHEMY_DATABASE_URI']
|
||||
|
||||
run_migration(args.source, target, args.dry_run)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,199 +0,0 @@
|
||||
"""
|
||||
Migrate USB checkout data from legacy database.
|
||||
|
||||
This script migrates USB device and checkout data from the VBScript database
|
||||
to the new USB plugin schema.
|
||||
|
||||
Usage:
|
||||
python -m scripts.migration.migrate_usb --source <connection_string>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_device_type_mapping(target_session):
|
||||
"""Get mapping of type names to IDs in target database."""
|
||||
result = target_session.execute(text(
|
||||
"SELECT usbdevicetypeid, typename FROM usbdevicetypes"
|
||||
))
|
||||
return {row.typename.lower(): row.usbdevicetypeid for row in result}
|
||||
|
||||
|
||||
def run_migration(source_conn_str, target_conn_str, dry_run=False):
|
||||
"""
|
||||
Run USB device migration.
|
||||
|
||||
Args:
|
||||
source_conn_str: Source database connection string
|
||||
target_conn_str: Target database connection string
|
||||
dry_run: If True, don't commit changes
|
||||
"""
|
||||
source_engine = create_engine(source_conn_str)
|
||||
target_engine = create_engine(target_conn_str)
|
||||
|
||||
SourceSession = sessionmaker(bind=source_engine)
|
||||
TargetSession = sessionmaker(bind=target_engine)
|
||||
|
||||
source_session = SourceSession()
|
||||
target_session = TargetSession()
|
||||
|
||||
try:
|
||||
# Get type mappings
|
||||
type_mapping = get_device_type_mapping(target_session)
|
||||
default_type_id = type_mapping.get('flash drive', 1)
|
||||
|
||||
# Migrate USB devices
|
||||
# Adjust table/column names based on actual legacy schema
|
||||
logger.info("Migrating USB devices...")
|
||||
|
||||
try:
|
||||
devices = source_session.execute(text("""
|
||||
SELECT * FROM usbdevices
|
||||
"""))
|
||||
|
||||
device_id_map = {} # Map old IDs to new IDs
|
||||
|
||||
for device in devices:
|
||||
device_dict = dict(device._mapping)
|
||||
|
||||
# Determine device type
|
||||
type_name = (device_dict.get('typename') or 'flash drive').lower()
|
||||
type_id = type_mapping.get(type_name, default_type_id)
|
||||
|
||||
result = target_session.execute(text("""
|
||||
INSERT INTO usbdevices (
|
||||
serialnumber, label, assetnumber, usbdevicetypeid,
|
||||
capacitygb, vendorid, productid, manufacturer, productname,
|
||||
ischeckedout, currentuserid, currentusername,
|
||||
storagelocation, notes, isactive, createddate
|
||||
) VALUES (
|
||||
:serialnumber, :label, :assetnumber, :usbdevicetypeid,
|
||||
:capacitygb, :vendorid, :productid, :manufacturer, :productname,
|
||||
:ischeckedout, :currentuserid, :currentusername,
|
||||
:storagelocation, :notes, :isactive, :createddate
|
||||
)
|
||||
"""), {
|
||||
'serialnumber': device_dict.get('serialnumber', f"UNKNOWN_{device_dict.get('usbdeviceid', 0)}"),
|
||||
'label': device_dict.get('label'),
|
||||
'assetnumber': device_dict.get('assetnumber'),
|
||||
'usbdevicetypeid': type_id,
|
||||
'capacitygb': device_dict.get('capacitygb'),
|
||||
'vendorid': device_dict.get('vendorid'),
|
||||
'productid': device_dict.get('productid'),
|
||||
'manufacturer': device_dict.get('manufacturer'),
|
||||
'productname': device_dict.get('productname'),
|
||||
'ischeckedout': device_dict.get('ischeckedout', False),
|
||||
'currentuserid': device_dict.get('currentuserid'),
|
||||
'currentusername': device_dict.get('currentusername'),
|
||||
'storagelocation': device_dict.get('storagelocation'),
|
||||
'notes': device_dict.get('notes'),
|
||||
'isactive': device_dict.get('isactive', True),
|
||||
'createddate': device_dict.get('createddate', datetime.utcnow()),
|
||||
})
|
||||
|
||||
# Get the new ID
|
||||
new_id = target_session.execute(text("SELECT LAST_INSERT_ID()")).scalar()
|
||||
device_id_map[device_dict.get('usbdeviceid')] = new_id
|
||||
|
||||
logger.info(f"Migrated {len(device_id_map)} USB devices")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not migrate USB devices: {e}")
|
||||
device_id_map = {}
|
||||
|
||||
# Migrate checkout history
|
||||
logger.info("Migrating USB checkout history...")
|
||||
|
||||
try:
|
||||
checkouts = source_session.execute(text("""
|
||||
SELECT * FROM usbcheckouts
|
||||
"""))
|
||||
|
||||
checkout_count = 0
|
||||
|
||||
for checkout in checkouts:
|
||||
checkout_dict = dict(checkout._mapping)
|
||||
|
||||
old_device_id = checkout_dict.get('usbdeviceid')
|
||||
new_device_id = device_id_map.get(old_device_id)
|
||||
|
||||
if not new_device_id:
|
||||
logger.warning(f"Skipping checkout - device ID {old_device_id} not found in mapping")
|
||||
continue
|
||||
|
||||
target_session.execute(text("""
|
||||
INSERT INTO usbcheckouts (
|
||||
usbdeviceid, userid, username,
|
||||
checkoutdate, checkindate, expectedreturndate,
|
||||
purpose, notes, checkedoutby, checkedinby,
|
||||
isactive, createddate
|
||||
) VALUES (
|
||||
:usbdeviceid, :userid, :username,
|
||||
:checkoutdate, :checkindate, :expectedreturndate,
|
||||
:purpose, :notes, :checkedoutby, :checkedinby,
|
||||
:isactive, :createddate
|
||||
)
|
||||
"""), {
|
||||
'usbdeviceid': new_device_id,
|
||||
'userid': checkout_dict.get('userid', 'unknown'),
|
||||
'username': checkout_dict.get('username'),
|
||||
'checkoutdate': checkout_dict.get('checkoutdate', datetime.utcnow()),
|
||||
'checkindate': checkout_dict.get('checkindate'),
|
||||
'expectedreturndate': checkout_dict.get('expectedreturndate'),
|
||||
'purpose': checkout_dict.get('purpose'),
|
||||
'notes': checkout_dict.get('notes'),
|
||||
'checkedoutby': checkout_dict.get('checkedoutby'),
|
||||
'checkedinby': checkout_dict.get('checkedinby'),
|
||||
'isactive': True,
|
||||
'createddate': checkout_dict.get('createddate', datetime.utcnow()),
|
||||
})
|
||||
|
||||
checkout_count += 1
|
||||
|
||||
logger.info(f"Migrated {checkout_count} checkout records")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not migrate USB checkouts: {e}")
|
||||
|
||||
if dry_run:
|
||||
logger.info("Dry run - rolling back changes")
|
||||
target_session.rollback()
|
||||
else:
|
||||
target_session.commit()
|
||||
|
||||
logger.info("USB migration complete")
|
||||
|
||||
finally:
|
||||
source_session.close()
|
||||
target_session.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Migrate USB devices and checkouts')
|
||||
parser.add_argument('--source', required=True, help='Source database connection string')
|
||||
parser.add_argument('--target', help='Target database connection string')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target = args.target
|
||||
if not target:
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
from shopdb import create_app
|
||||
app = create_app()
|
||||
target = app.config['SQLALCHEMY_DATABASE_URI']
|
||||
|
||||
run_migration(args.source, target, args.dry_run)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,139 +0,0 @@
|
||||
"""
|
||||
Migration orchestrator - runs all migration steps in order.
|
||||
|
||||
This script coordinates the full migration from VBScript ShopDB to Flask.
|
||||
|
||||
Usage:
|
||||
python -m scripts.migration.run_migration --source <connection_string>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_full_migration(source_conn_str, target_conn_str, dry_run=False, steps=None):
|
||||
"""
|
||||
Run the full migration process.
|
||||
|
||||
Args:
|
||||
source_conn_str: Source database connection string
|
||||
target_conn_str: Target database connection string
|
||||
dry_run: If True, don't commit changes
|
||||
steps: List of specific steps to run, or None for all
|
||||
"""
|
||||
from . import migrate_assets
|
||||
from . import migrate_communications
|
||||
from . import migrate_notifications
|
||||
from . import migrate_usb
|
||||
from . import verify_migration
|
||||
|
||||
all_steps = [
|
||||
('assets', 'Migrate machines to assets', migrate_assets.run_migration),
|
||||
('communications', 'Update communications FKs', migrate_communications.run_migration),
|
||||
('notifications', 'Migrate notifications', migrate_notifications.run_migration),
|
||||
('usb', 'Migrate USB devices', migrate_usb.run_migration),
|
||||
]
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("SHOPDB MIGRATION")
|
||||
logger.info(f"Started: {datetime.utcnow().isoformat()}")
|
||||
logger.info(f"Dry Run: {dry_run}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
results = {}
|
||||
|
||||
for step_name, description, migration_func in all_steps:
|
||||
if steps and step_name not in steps:
|
||||
logger.info(f"\nSkipping: {description}")
|
||||
continue
|
||||
|
||||
logger.info(f"\n{'=' * 40}")
|
||||
logger.info(f"Step: {description}")
|
||||
logger.info('=' * 40)
|
||||
|
||||
try:
|
||||
# Different migrations have different signatures
|
||||
if step_name == 'communications':
|
||||
migration_func(target_conn_str, dry_run)
|
||||
else:
|
||||
migration_func(source_conn_str, target_conn_str, dry_run)
|
||||
|
||||
results[step_name] = 'SUCCESS'
|
||||
logger.info(f"Step completed: {step_name}")
|
||||
|
||||
except Exception as e:
|
||||
results[step_name] = f'FAILED: {e}'
|
||||
logger.error(f"Step failed: {step_name} - {e}")
|
||||
|
||||
# Ask to continue
|
||||
if not dry_run:
|
||||
response = input("Continue with next step? (y/n): ")
|
||||
if response.lower() != 'y':
|
||||
logger.info("Migration aborted by user")
|
||||
break
|
||||
|
||||
# Run verification
|
||||
logger.info(f"\n{'=' * 40}")
|
||||
logger.info("Running verification...")
|
||||
logger.info('=' * 40)
|
||||
|
||||
try:
|
||||
verify_migration.run_verification(source_conn_str, target_conn_str)
|
||||
results['verification'] = 'SUCCESS'
|
||||
except Exception as e:
|
||||
results['verification'] = f'FAILED: {e}'
|
||||
logger.error(f"Verification failed: {e}")
|
||||
|
||||
# Summary
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("MIGRATION SUMMARY")
|
||||
logger.info("=" * 60)
|
||||
|
||||
for step, result in results.items():
|
||||
status = "OK" if result == 'SUCCESS' else "FAILED"
|
||||
logger.info(f" {step}: {status}")
|
||||
if result != 'SUCCESS':
|
||||
logger.info(f" {result}")
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Completed: {datetime.utcnow().isoformat()}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Run full ShopDB migration')
|
||||
parser.add_argument('--source', required=True, help='Source database connection string')
|
||||
parser.add_argument('--target', help='Target database connection string')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Dry run without committing')
|
||||
parser.add_argument('--steps', nargs='+',
|
||||
choices=['assets', 'communications', 'notifications', 'usb'],
|
||||
help='Specific steps to run')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target = args.target
|
||||
if not target:
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
from shopdb import create_app
|
||||
app = create_app()
|
||||
target = app.config['SQLALCHEMY_DATABASE_URI']
|
||||
|
||||
results = run_full_migration(args.source, target, args.dry_run, args.steps)
|
||||
|
||||
# Exit with error if any step failed
|
||||
if any(r != 'SUCCESS' for r in results.values()):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,174 +0,0 @@
|
||||
"""
|
||||
Verify data migration integrity.
|
||||
|
||||
This script compares record counts between source and target databases
|
||||
and performs spot checks on data integrity.
|
||||
|
||||
Usage:
|
||||
python -m scripts.migration.verify_migration --source <connection_string>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def verify_counts(source_session, target_session):
|
||||
"""Compare record counts between source and target."""
|
||||
results = {}
|
||||
|
||||
# Define table mappings (source -> target)
|
||||
table_mappings = [
|
||||
('machines', 'assets', 'Machine to Asset'),
|
||||
('communications', 'communications', 'Communications'),
|
||||
('vendors', 'vendors', 'Vendors'),
|
||||
('locations', 'locations', 'Locations'),
|
||||
('businessunits', 'businessunits', 'Business Units'),
|
||||
]
|
||||
|
||||
for source_table, target_table, description in table_mappings:
|
||||
try:
|
||||
source_count = source_session.execute(text(f"SELECT COUNT(*) FROM {source_table}")).scalar()
|
||||
except Exception as e:
|
||||
source_count = f"Error: {e}"
|
||||
|
||||
try:
|
||||
target_count = target_session.execute(text(f"SELECT COUNT(*) FROM {target_table}")).scalar()
|
||||
except Exception as e:
|
||||
target_count = f"Error: {e}"
|
||||
|
||||
match = source_count == target_count if isinstance(source_count, int) and isinstance(target_count, int) else False
|
||||
|
||||
results[description] = {
|
||||
'source': source_count,
|
||||
'target': target_count,
|
||||
'match': match
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def verify_sample_records(source_session, target_session, sample_size=10):
|
||||
"""Spot-check sample records for data integrity."""
|
||||
issues = []
|
||||
|
||||
# Sample machine -> asset migration
|
||||
try:
|
||||
sample_machines = source_session.execute(text(f"""
|
||||
SELECT machineid, machinenumber, serialnumber, alias
|
||||
FROM machines
|
||||
ORDER BY RAND()
|
||||
LIMIT {sample_size}
|
||||
"""))
|
||||
|
||||
for machine in sample_machines:
|
||||
machine_dict = dict(machine._mapping)
|
||||
|
||||
# Check if asset exists with same ID
|
||||
asset = target_session.execute(text("""
|
||||
SELECT assetid, assetnumber, serialnumber, name
|
||||
FROM assets
|
||||
WHERE assetid = :assetid
|
||||
"""), {'assetid': machine_dict['machineid']}).fetchone()
|
||||
|
||||
if not asset:
|
||||
issues.append(f"Machine {machine_dict['machineid']} not found in assets")
|
||||
continue
|
||||
|
||||
asset_dict = dict(asset._mapping)
|
||||
|
||||
# Verify data matches
|
||||
if machine_dict['machinenumber'] != asset_dict['assetnumber']:
|
||||
issues.append(f"Asset {asset_dict['assetid']}: machinenumber mismatch")
|
||||
if machine_dict.get('serialnumber') != asset_dict.get('serialnumber'):
|
||||
issues.append(f"Asset {asset_dict['assetid']}: serialnumber mismatch")
|
||||
|
||||
except Exception as e:
|
||||
issues.append(f"Could not verify machines: {e}")
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def run_verification(source_conn_str, target_conn_str):
|
||||
"""
|
||||
Run migration verification.
|
||||
|
||||
Args:
|
||||
source_conn_str: Source database connection string
|
||||
target_conn_str: Target database connection string
|
||||
"""
|
||||
source_engine = create_engine(source_conn_str)
|
||||
target_engine = create_engine(target_conn_str)
|
||||
|
||||
SourceSession = sessionmaker(bind=source_engine)
|
||||
TargetSession = sessionmaker(bind=target_engine)
|
||||
|
||||
source_session = SourceSession()
|
||||
target_session = TargetSession()
|
||||
|
||||
try:
|
||||
logger.info("=" * 60)
|
||||
logger.info("MIGRATION VERIFICATION REPORT")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Verify counts
|
||||
logger.info("\nRecord Count Comparison:")
|
||||
logger.info("-" * 40)
|
||||
counts = verify_counts(source_session, target_session)
|
||||
|
||||
all_match = True
|
||||
for table, result in counts.items():
|
||||
status = "OK" if result['match'] else "MISMATCH"
|
||||
if not result['match']:
|
||||
all_match = False
|
||||
logger.info(f" {table}: Source={result['source']}, Target={result['target']} [{status}]")
|
||||
|
||||
# Verify sample records
|
||||
logger.info("\nSample Record Verification:")
|
||||
logger.info("-" * 40)
|
||||
issues = verify_sample_records(source_session, target_session)
|
||||
|
||||
if issues:
|
||||
for issue in issues:
|
||||
logger.warning(f" ! {issue}")
|
||||
else:
|
||||
logger.info(" All sample records verified OK")
|
||||
|
||||
# Summary
|
||||
logger.info("\n" + "=" * 60)
|
||||
if all_match and not issues:
|
||||
logger.info("VERIFICATION PASSED - Migration looks good!")
|
||||
else:
|
||||
logger.warning("VERIFICATION FOUND ISSUES - Review above")
|
||||
logger.info("=" * 60)
|
||||
|
||||
finally:
|
||||
source_session.close()
|
||||
target_session.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Verify migration integrity')
|
||||
parser.add_argument('--source', required=True, help='Source database connection string')
|
||||
parser.add_argument('--target', help='Target database connection string')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target = args.target
|
||||
if not target:
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
from shopdb import create_app
|
||||
app = create_app()
|
||||
target = app.config['SQLALCHEMY_DATABASE_URI']
|
||||
|
||||
run_verification(args.source, target)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user