Add site asset CSV import with fuzzy reconciliation
Sites seed inventory from the Asset Collector script or spreadsheet exports, but lookup names drift between sources (Dell vs Dell Inc.), which would otherwise create duplicate vendors/models/types on every import. The importer now normalizes and fuzzy-matches near-miss references onto existing lookup rows within an operator-set similarity threshold, auto-applying the match while reporting it for review so nothing merges silently. Operating systems and asset rows are never reconciled. Adds the dedicated Import Site Assets page with script/template download and a multi-file dry-run/commit workflow.
This commit is contained in:
benjamin.baker
@@ -1,6 +1,7 @@
|
||||
"""Settings API routes."""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from flask import Blueprint, request, current_app
|
||||
from flask_jwt_extended import jwt_required, get_jwt_identity
|
||||
@@ -240,6 +241,218 @@ def serve_branding_logo(filename):
|
||||
return send_upload(_branding_dir(), filename)
|
||||
|
||||
|
||||
def _asset_import_dir():
|
||||
"""Configured Asset Collector CSV location, or '' when unset."""
|
||||
setting = Setting.query.filter_by(key='asset_import_csv_location').first()
|
||||
return (setting.value or '').strip() if setting else ''
|
||||
|
||||
|
||||
# The collector script that gets served (with the site's CSV path baked in) from
|
||||
# the Import Site Assets page. Copied into the image under scripts/ (see
|
||||
# Dockerfile), so it is always available next to the running API.
|
||||
_COLLECTOR_SCRIPT = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
|
||||
'scripts', 'site_imports', 'assetcollector', 'Collect-ShopDB.ps1')
|
||||
|
||||
|
||||
def _embed_store_path(script_text, path):
|
||||
"""Return the collector script with $NetworkStoreRoot set to path.
|
||||
|
||||
The script ships with a site-specific default on that line; rewriting it
|
||||
means an operator downloads a script already pointed at this facility's CSV
|
||||
store and does not have to edit PowerShell by hand. A blank path leaves the
|
||||
shipped default untouched.
|
||||
"""
|
||||
if not path:
|
||||
return script_text
|
||||
literal = path.replace("'", "''")
|
||||
replacement = "$NetworkStoreRoot = '%s'" % literal
|
||||
# A function replacement is used deliberately: a UNC path is full of
|
||||
# backslashes, and a string replacement runs them through re's escape
|
||||
# processing (so \s, \w, a trailing \1 all raise or corrupt). A callable
|
||||
# returns the text verbatim.
|
||||
return re.sub(
|
||||
r"(?m)^\$NetworkStoreRoot\s*=.*$",
|
||||
lambda _match: replacement,
|
||||
script_text,
|
||||
count=1)
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/script', methods=['GET'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def download_collector_script():
|
||||
"""Download Collect-ShopDB.ps1 with this site's CSV path embedded."""
|
||||
try:
|
||||
with open(_COLLECTOR_SCRIPT, 'r', encoding='utf-8') as handle:
|
||||
script_text = handle.read()
|
||||
except OSError:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
'Collector script is not available on the server',
|
||||
http_code=404)
|
||||
|
||||
script_text = _embed_store_path(script_text, _asset_import_dir())
|
||||
response = current_app.response_class(script_text, mimetype='text/plain')
|
||||
response.headers['Content-Disposition'] = 'attachment; filename=Collect-ShopDB.ps1'
|
||||
return response
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/templates', methods=['GET'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def list_import_templates():
|
||||
"""List the importable CSV tables in the order they must be uploaded."""
|
||||
from shopdb.core.services.csvimport import IMPORTABLE
|
||||
|
||||
tables = [{'table': name, 'filename': '%s.csv' % name}
|
||||
for name in IMPORTABLE]
|
||||
return success_response({'tables': tables})
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/templates/<tablename>', methods=['GET'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def download_import_template(tablename):
|
||||
"""Download a generated CSV template for a single importable table."""
|
||||
from shopdb.core.services.csvimport import (
|
||||
IMPORTABLE, generate_template, ImportError_)
|
||||
|
||||
if tablename not in IMPORTABLE:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
'No such importable table: %s' % tablename,
|
||||
http_code=404)
|
||||
try:
|
||||
template_text = generate_template(tablename)
|
||||
except ImportError_ as exc:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
|
||||
|
||||
response = current_app.response_class(template_text, mimetype='text/csv')
|
||||
response.headers['Content-Disposition'] = (
|
||||
'attachment; filename=%s.csv' % tablename)
|
||||
return response
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/import', methods=['POST'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def import_asset_csvs():
|
||||
"""Validate uploaded CSVs and, when commit=true, load them.
|
||||
|
||||
Mirrors the 'flask csv import' CLI: every file runs through import_csv inside
|
||||
ONE transaction with a shared resolver, so a computers.csv can resolve the
|
||||
assets.csv uploaded alongside it. A dry run (the default) rolls the whole
|
||||
transaction back and reports what WOULD change; commit=true persists it. The
|
||||
whole run is all-or-nothing - if any file has a row problem, nothing is
|
||||
written even on a commit.
|
||||
"""
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from shopdb.core.services.csvimport import (
|
||||
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
|
||||
|
||||
uploads = request.files.getlist('files')
|
||||
if not uploads:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR, 'No CSV files were uploaded')
|
||||
|
||||
commit = str(request.form.get('commit', '')).strip().lower() in (
|
||||
'1', 'true', 'yes', 'on')
|
||||
|
||||
# Map each upload to a table by filename. A later file with the same table
|
||||
# name wins, matching the folder-import behaviour (one file per table).
|
||||
texts = {}
|
||||
filenames = {}
|
||||
for upload in uploads:
|
||||
name = upload.filename or ''
|
||||
if not name.lower().endswith('.csv'):
|
||||
continue
|
||||
table = table_from_filename(name)
|
||||
try:
|
||||
texts[table] = upload.read().decode('utf-8-sig')
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR,
|
||||
'Could not read %s as UTF-8 text' % name)
|
||||
filenames[table] = name
|
||||
|
||||
if not texts:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR,
|
||||
'None of the uploaded files were CSV files')
|
||||
|
||||
ordered = dependency_order(list(texts))
|
||||
skipped = sorted(set(texts) - set(ordered))
|
||||
|
||||
reconcile_setting = Setting.query.filter_by(
|
||||
key='asset_import_reconcile_threshold').first()
|
||||
try:
|
||||
reconcile_threshold = float(reconcile_setting.value) if reconcile_setting and reconcile_setting.value else 0.0
|
||||
except (TypeError, ValueError):
|
||||
reconcile_threshold = 0.0
|
||||
|
||||
resolver = Resolver(reconcile_threshold=reconcile_threshold)
|
||||
files_report = []
|
||||
problem_count = 0
|
||||
reconciled_count = 0
|
||||
request_error = None
|
||||
|
||||
try:
|
||||
for table in ordered:
|
||||
try:
|
||||
result = import_csv(table, texts[table], resolver=resolver, commit=False)
|
||||
except ImportError_ as exc:
|
||||
request_error = '%s: %s' % (filenames[table], str(exc))
|
||||
break
|
||||
except SQLAlchemyError as exc:
|
||||
db.session.rollback()
|
||||
request_error = '%s: database error: %s' % (
|
||||
filenames[table], str(exc).split(chr(10))[0])
|
||||
break
|
||||
|
||||
problems = [
|
||||
{'line': p.line, 'column': p.column, 'message': p.message}
|
||||
for p in result.problems
|
||||
]
|
||||
problem_count += len(problems)
|
||||
reconciled_count += len(result.reconciliations)
|
||||
files_report.append({
|
||||
'table': table,
|
||||
'filename': filenames[table],
|
||||
'created': result.created,
|
||||
'updated': result.updated,
|
||||
'problems': problems,
|
||||
'reconciliations': result.reconciliations,
|
||||
})
|
||||
|
||||
applied = False
|
||||
if request_error is not None:
|
||||
db.session.rollback()
|
||||
elif problem_count > 0:
|
||||
# A row problem anywhere fails the whole run, so nothing is written
|
||||
# even when commit was asked for.
|
||||
db.session.rollback()
|
||||
elif commit:
|
||||
db.session.commit()
|
||||
applied = True
|
||||
else:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
total_created = sum(f['created'] for f in files_report)
|
||||
total_updated = sum(f['updated'] for f in files_report)
|
||||
|
||||
return success_response({
|
||||
'committed': applied,
|
||||
'requested_commit': commit,
|
||||
'request_error': request_error,
|
||||
'files': files_report,
|
||||
'skipped': skipped,
|
||||
'problem_count': problem_count,
|
||||
'reconciled_count': reconciled_count,
|
||||
'reconcile_threshold': reconcile_threshold,
|
||||
'total_created': total_created,
|
||||
'total_updated': total_updated,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route('', methods=['GET'])
|
||||
@jwt_required(optional=True)
|
||||
def list_settings():
|
||||
@@ -556,6 +769,20 @@ def build_default_settings():
|
||||
'category': 'site',
|
||||
'description': 'Email domain appended to a contact SSO to build email/Teams links (sso@domain). Blank disables contact action buttons.'
|
||||
},
|
||||
{
|
||||
'key': 'asset_import_csv_location',
|
||||
'value': '',
|
||||
'valuetype': 'string',
|
||||
'category': 'site',
|
||||
'description': r'Shared network location where the Asset Collector script deposits its CSVs and where ShopDB reads them for import (e.g. \\server.fqdn\share\folder). This path is baked into the downloadable collector script. Blank = not configured.'
|
||||
},
|
||||
{
|
||||
'key': 'asset_import_reconcile_threshold',
|
||||
'value': '0.85',
|
||||
'valuetype': 'string',
|
||||
'category': 'site',
|
||||
'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.'
|
||||
},
|
||||
]
|
||||
|
||||
# Site branding. Each instance (ADR-004) can replace the shipped GE defaults
|
||||
|
||||
Reference in New Issue
Block a user