Cam Proudlock
8666e0271c
The machine page now shows the PC that controls it. The endpoint was correct all along and returned both directions; AssetTriageBand discarded every peer whose type was not 'machine' and was mounted only on the PC page. The counterpart type, label and relationship types are now props. Fixed while adopting it more widely: every band chip linked to /assets/<id>, and there is no such route. The resolver that AssetRelationships.vue kept privately is now frontend/src/utils/assetRoute.js, used by both - and its type lookup is case-insensitive, because seed data capitalises the type names while this site's database does not. The legacy import is now safe to resume. The 409-resolve asked for per_page=10000 against a server that clamps to 100, so it silently searched the first 100 rows and dropped links past that on every re-run. The crosswalk is now bound to the database it describes, so one carried across a rebuild is refused rather than attaching rows to the wrong assets. Source database names are resolved from the environment, so the loader can be aimed without editing it. Full suite 2178 passed, frontend 280 passed.
307 lines
13 KiB
Python
307 lines
13 KiB
Python
"""Import harness for the WJ classic-ASP -> flask reference loader.
|
|
|
|
This is SITE-SPECIFIC reference glue, NOT product code. It maps West Jefferson's
|
|
classic schema onto the maintained, schema-agnostic import contract in
|
|
the IMPORT-API wiki page. Other sites copy this pattern against their own source DB;
|
|
they do not run this loader.
|
|
|
|
The harness builds the flask app against the current DATABASE_URL (point it at a
|
|
throwaway import database), mints an unscoped admin PAT in-process, and drives
|
|
the REAL import endpoints through the app test client with the X-Import-Mode
|
|
header - so it exercises the same routes/authz/validation an HTTP client would,
|
|
without needing a running server. Legacy-id -> new-id maps persist to JSON so a
|
|
crashed run resumes and later stages resolve foreign keys.
|
|
"""
|
|
|
|
import json
|
|
import uuid
|
|
import os
|
|
import pymysql
|
|
|
|
from shopdb import create_app
|
|
from shopdb.extensions import db
|
|
|
|
|
|
def _source_password():
|
|
"""Root password for the scratch source DBs: reuse the dev DATABASE_URL in
|
|
.env unless SOURCE_DB_PASSWORD overrides."""
|
|
if os.environ.get('SOURCE_DB_PASSWORD'):
|
|
return os.environ['SOURCE_DB_PASSWORD']
|
|
env = os.path.join(os.path.dirname(__file__), '..', '..', '..', '.env')
|
|
with open(os.path.abspath(env)) as handle:
|
|
for line in handle:
|
|
if line.startswith('DATABASE_URL='):
|
|
# Not every server runs as root, and 'root:' then simply is not
|
|
# there: split()[1] raised IndexError, which read as a crash
|
|
# rather than the clear message two lines down.
|
|
if 'root:' not in line:
|
|
continue
|
|
after = line.split('root:', 1)[1]
|
|
return after.split('@', 1)[0]
|
|
raise RuntimeError('could not resolve source DB password')
|
|
|
|
|
|
class Source:
|
|
"""Read-only access to the three source databases.
|
|
|
|
The stage functions name a source by its LOGICAL name - 'shopdb_src',
|
|
'cmmc_usb_src', 'wjf_employees_src' - and this class resolves each to a
|
|
real database. That indirection is the point: the names were hardcoded at
|
|
29 call sites, so pointing the loader at a differently-named copy (the
|
|
prodscratch restore, or a site's own dump) meant editing run.py, which is
|
|
not something anybody should do to aim a loader. The logical names stay in
|
|
the stages where they read as documentation; the mapping lives here.
|
|
|
|
Everything is environment-overridable, defaulting to the scratch layout:
|
|
|
|
SOURCE_DB_MAIN default shopdb_src
|
|
SOURCE_DB_USB default cmmc_usb_src
|
|
SOURCE_DB_EMPLOYEES default wjf_employees_src
|
|
SOURCE_DB_HOST/PORT/USER/PASSWORD
|
|
"""
|
|
|
|
# logical name -> (env var, default real name)
|
|
ALIASES = {
|
|
'shopdb_src': ('SOURCE_DB_MAIN', 'shopdb_src'),
|
|
'cmmc_usb_src': ('SOURCE_DB_USB', 'cmmc_usb_src'),
|
|
'wjf_employees_src': ('SOURCE_DB_EMPLOYEES', 'wjf_employees_src'),
|
|
}
|
|
|
|
def __init__(self, host=None, port=None, user=None, password=None):
|
|
self.password = password or _source_password()
|
|
self.host = host or os.environ.get('SOURCE_DB_HOST', '127.0.0.1')
|
|
self.port = int(port or os.environ.get('SOURCE_DB_PORT', 3306))
|
|
self.user = user or os.environ.get('SOURCE_DB_USER', 'root')
|
|
self._conns = {}
|
|
|
|
@classmethod
|
|
def resolve(cls, name):
|
|
"""The real database behind a logical source name."""
|
|
env, default = cls.ALIASES.get(name, (None, name))
|
|
return os.environ.get(env, default) if env else name
|
|
|
|
def db(self, name):
|
|
real = self.resolve(name)
|
|
if real not in self._conns:
|
|
self._conns[real] = pymysql.connect(
|
|
host=self.host, port=self.port, user=self.user,
|
|
password=self.password, database=real, charset='utf8mb4',
|
|
cursorclass=pymysql.cursors.DictCursor)
|
|
return self._conns[real]
|
|
|
|
def rows(self, dbname, sql, args=None):
|
|
with self.db(dbname).cursor() as cur:
|
|
cur.execute(sql, args or ())
|
|
return cur.fetchall()
|
|
|
|
|
|
class StaleIdMapError(RuntimeError):
|
|
"""The crosswalk on disk was built against a different target database."""
|
|
|
|
|
|
class IdMap:
|
|
"""Persisted legacy-id -> new-id crosswalks, one namespace per entity.
|
|
|
|
BOUND TO ONE TARGET DATABASE. The crosswalk holds this site's legacy ids
|
|
mapped to assetids in a specific database, and those assetids mean nothing
|
|
anywhere else. Two ways that used to go wrong, both silent:
|
|
|
|
- Delete idmap.json and re-run: every create 409s, nothing resolves into
|
|
the crosswalk, and warranties, installs, relationships and printer
|
|
assignments all quietly import as ~0 rows.
|
|
- Keep idmap.json across a drop-and-rebuild of the target: the legacy ids
|
|
now point at assetids belonging to the OLD database, so warranties and
|
|
printer assignments attach to whatever asset happens to hold that id
|
|
now. Wrong data, no error, and the runbook's two "drop and rebuild"
|
|
steps never said to delete the file.
|
|
|
|
The bind is a marker row in the target's own settings table, so it dies
|
|
with the data: a rebuild produces a new marker, and the mismatch is caught
|
|
before a single row is written. The database NAME is not enough on its own,
|
|
because a rebuild keeps it.
|
|
"""
|
|
|
|
MARKER_KEY = 'siteimport_targetinstance'
|
|
|
|
def __init__(self, path, target=None):
|
|
self.path = path
|
|
self.target = target
|
|
self._data = {}
|
|
if os.path.exists(path):
|
|
with open(path) as handle:
|
|
self._data = json.load(handle)
|
|
self._check_target()
|
|
|
|
def _check_target(self):
|
|
if not self.target:
|
|
return
|
|
seen = self._data.get('_target')
|
|
if seen is None:
|
|
# Written before this check existed. Adopt it rather than refuse:
|
|
# refusing would strand a crosswalk that is probably fine, and the
|
|
# next run is bound properly.
|
|
self._data['_target'] = self.target
|
|
self.save()
|
|
return
|
|
if seen != self.target:
|
|
raise StaleIdMapError(
|
|
'%s was built against target %s, but this run targets %s.\n'
|
|
'Those assetids belong to a different database and using them '
|
|
'would attach imported rows to the wrong assets.\n'
|
|
'Delete the file to start a fresh crosswalk, or point '
|
|
'--idmap at the one for this target.'
|
|
% (self.path, seen, self.target))
|
|
|
|
def put(self, namespace, legacy, new):
|
|
self._data.setdefault(namespace, {})[str(legacy)] = new
|
|
self.save()
|
|
|
|
def get(self, namespace, legacy, default=None):
|
|
return self._data.get(namespace, {}).get(str(legacy), default)
|
|
|
|
def has(self, namespace, legacy):
|
|
return str(legacy) in self._data.get(namespace, {})
|
|
|
|
def count(self, namespace):
|
|
entry = self._data.get(namespace, {})
|
|
return len(entry) if isinstance(entry, dict) else 0
|
|
|
|
def save(self):
|
|
with open(self.path, 'w') as handle:
|
|
json.dump(self._data, handle, indent=1)
|
|
|
|
|
|
class Harness:
|
|
"""App + admin PAT + import-mode test client + id maps."""
|
|
|
|
def __init__(self, idmap_path=None):
|
|
self.app = create_app()
|
|
self.appcontext = self.app.app_context()
|
|
self.appcontext.push()
|
|
self._silence_sql_logging()
|
|
self.client = self.app.test_client()
|
|
self._ensure_asset_types()
|
|
self.secret = self._mint_admin_pat()
|
|
default = os.path.join(os.path.dirname(__file__), 'idmap.json')
|
|
self.ids = IdMap(idmap_path or default, target=self._target_instance())
|
|
self.source = Source()
|
|
self.errors = []
|
|
self._defaultlevelid = None
|
|
|
|
def _target_instance(self):
|
|
"""A stable id for THIS target database, created once and stored in it.
|
|
|
|
Kept in the target's own settings table so it dies when the data does.
|
|
A drop-and-rebuild yields a new id, which is what lets IdMap catch a
|
|
crosswalk carried over from the previous database - the failure that
|
|
silently attaches imported rows to whatever asset now holds that id.
|
|
|
|
The database name alone would not do: a rebuild keeps the name.
|
|
"""
|
|
from shopdb.core.models import Setting
|
|
existing = Setting.query.filter_by(key=IdMap.MARKER_KEY).first()
|
|
if existing and existing.value:
|
|
return existing.value
|
|
marker = uuid.uuid4().hex
|
|
if existing:
|
|
existing.value = marker
|
|
else:
|
|
db.session.add(Setting(
|
|
key=IdMap.MARKER_KEY, value=marker, valuetype='string',
|
|
category='import',
|
|
description='Identifies this database to the site-import '
|
|
'crosswalk. Created by the loader; do not copy it '
|
|
'between databases.'))
|
|
db.session.commit()
|
|
return marker
|
|
|
|
@property
|
|
def defaultlevelid(self):
|
|
"""The level imported map positions belong to (ADR-017).
|
|
|
|
The legacy schema predates levels: it has ONE floor plan, so every
|
|
mapleft/maptop it carries is a coordinate on this site's default level.
|
|
Importing them without a level produces markers the map refuses to draw
|
|
- it will not guess a drawing for coordinates that do not name one.
|
|
"""
|
|
if self._defaultlevelid is None:
|
|
from shopdb.core.models import MapLevel
|
|
level = MapLevel.default_level()
|
|
self._defaultlevelid = level.levelid if level else None
|
|
return self._defaultlevelid
|
|
|
|
def _silence_sql_logging(self):
|
|
import logging
|
|
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)
|
|
|
|
def _ensure_asset_types(self):
|
|
"""Register each plugin's AssetType + seeded subtypes. on_install (which
|
|
does this) is skipped when a DB is built with `flask plugin upgrade-all`
|
|
rather than a fresh `install`, so run it idempotently here - the create
|
|
routes 500 without the AssetType row."""
|
|
manager = self.app.extensions.get('plugin_manager')
|
|
if not manager:
|
|
return
|
|
for _name, plugin in manager.get_all_plugins().items():
|
|
try:
|
|
plugin.on_install(self.app)
|
|
except Exception:
|
|
pass
|
|
# A full import IS the setup, so mark it complete - otherwise the admin
|
|
# is bounced to the first-run wizard on every fresh import DB.
|
|
from shopdb.core.models import Setting
|
|
Setting.set('setup_complete', 'true', valuetype='boolean', category='site')
|
|
db.session.commit()
|
|
|
|
def _mint_admin_pat(self):
|
|
"""Create (or reuse) an admin user and an unscoped admin PAT. Unscoped
|
|
because import mode + role gates require full owner authority
|
|
(the IMPORT-API wiki page). Returns the clear secret."""
|
|
from werkzeug.security import generate_password_hash
|
|
from shopdb.core.models import User, Role, ApiToken
|
|
admin = User.query.join(User.roles).filter(Role.rolename == 'admin').first()
|
|
if not admin:
|
|
role = Role.query.filter_by(rolename='admin').first() \
|
|
or Role(rolename='admin', description='Administrator')
|
|
db.session.add(role)
|
|
admin = User(username='importer', email='importer@localhost',
|
|
passwordhash=generate_password_hash('x'), isactive=True)
|
|
admin.roles.append(role)
|
|
db.session.add(admin)
|
|
db.session.commit()
|
|
secret = ApiToken.generate_secret()
|
|
token = ApiToken(userid=admin.userid, name='wjf legacy import',
|
|
tokenprefix=ApiToken.prefix_of(secret),
|
|
tokenhash=ApiToken.hash_secret(secret), scopes=None)
|
|
db.session.add(token)
|
|
db.session.commit()
|
|
return secret
|
|
|
|
def _headers(self, import_mode):
|
|
# A PAT authenticates as Authorization: Bearer shopdb_pat_... (the
|
|
# before_request shim resolves it into a JWT identity). X-API-Key is for
|
|
# collector/service tokens, not personal tokens.
|
|
headers = {'Authorization': f'Bearer {self.secret}'}
|
|
if import_mode:
|
|
headers['X-Import-Mode'] = 'true'
|
|
return headers
|
|
|
|
def post(self, path, payload, import_mode=True):
|
|
"""POST to an import endpoint. Returns (status, data). 409s are returned
|
|
(idempotency handled by the caller), other 4xx/5xx are recorded."""
|
|
resp = self.client.post(path, json=payload, headers=self._headers(import_mode))
|
|
body = resp.get_json() or {}
|
|
data = body.get('data', body)
|
|
if resp.status_code >= 400 and resp.status_code != 409:
|
|
self.errors.append((path, resp.status_code, payload, data))
|
|
return resp.status_code, data
|
|
|
|
def get(self, path):
|
|
resp = self.client.get(path, headers=self._headers(False))
|
|
body = resp.get_json() or {}
|
|
return resp.status_code, body.get('data', body)
|
|
|
|
def close(self):
|
|
self.ids.save()
|
|
self.appcontext.pop()
|