Add site asset CSV import with fuzzy reconciliation

Sites seed inventory from the Asset Collector script or spreadsheet exports, but lookup names drift between sources (Dell vs Dell Inc.), which would otherwise create duplicate vendors/models/types on every import. The importer now normalizes and fuzzy-matches near-miss references onto existing lookup rows within an operator-set similarity threshold, auto-applying the match while reporting it for review so nothing merges silently. Operating systems and asset rows are never reconciled. Adds the dedicated Import Site Assets page with script/template download and a multi-file dry-run/commit workflow.

shopdb/core/services/csvimport.py

@@ -34,6 +34,7 @@ not at all.
 from __future__ import annotations
 
 import csv
+import difflib
 import io
 import re
 from collections import OrderedDict
@@ -86,6 +87,43 @@ PASSWORD_COLUMNS = {'passwordhash', 'password'}
 TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'}
 FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''}
 
+# Reconciliation: when a name does not match a lookup row exactly, the resolver
+# may still bind it to an existing row that is close enough. Only lookup tables
+# take part - the small reference tables an operator picks values from. Rows that
+# ARE the data being imported (assets, computers, machines) and the fast-moving
+# operatingsystems list are excluded, so a typo there is a new row, not a silent
+# merge onto something unrelated.
+RECONCILABLE_TABLES = {
+    'assetstatuses', 'assettypes', 'locationtypes', 'modeltypes',
+    'computertypes', 'machinetypes', 'businessunits', 'locations',
+    'vendors', 'models',
+}
+
+# Corporate suffixes stripped before comparison so 'Dell Inc.' reconciles to
+# 'Dell'. Lowercase, punctuation already removed by the time these are checked.
+CORPORATE_SUFFIXES = {
+    'inc', 'incorporated', 'llc', 'llp', 'ltd', 'limited', 'corp',
+    'corporation', 'co', 'company', 'gmbh', 'ag', 'sa', 'plc',
+}
+
+# Default similarity a fuzzy match must reach, 0..1. Also the value the settings
+# default is seeded with; 0 disables reconciliation entirely.
+DEFAULT_RECONCILE_THRESHOLD = 0.85
+
+
+def normalize_label(value):
+    """Fold a label to its comparison form: lowercase, no punctuation, no
+    corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'.
+    """
+    text = (value or '').strip().lower()
+    text = re.sub(r'[^a-z0-9]+', ' ', text).strip()
+    if not text:
+        return ''
+    words = [w for w in text.split(' ') if w]
+    while len(words) > 1 and words[-1] in CORPORATE_SUFFIXES:
+        words.pop()
+    return ' '.join(words)
+
 
 def table_for(name):
     table = db.metadata.tables.get(name)
@@ -221,10 +259,17 @@ class Resolver:
 
     Rows created earlier in the SAME import are resolvable, so one file can
     reference another that was loaded moments ago without a commit in between.
+
+    When a name does not match exactly and the table is reconcilable, a
+    normalized then fuzzy comparison against the existing rows may bind it to a
+    close-enough match. Every such binding is recorded so the caller can report
+    it for review; nothing is merged silently.
     """
 
-    def __init__(self):
+    def __init__(self, reconcile_threshold=0.0):
         self._cache = {}
+        self.reconcile_threshold = reconcile_threshold or 0.0
+        self.events = []
 
     def resolve(self, tablename, idcolumn, value):
         if value is None:
@@ -249,6 +294,10 @@ class Resolver:
                 db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower())
         ).first()
         if row is None:
+            found = self._reconcile(tablename, idcolumn, label, table, value)
+            if found is not None:
+                self._cache[key] = found
+                return found
             raise ValueError(
                 "nothing in %s is named '%s' - add it to %s.csv, or import that file first"
                 % (tablename, value, tablename))
@@ -256,6 +305,52 @@ class Resolver:
         self._cache[key] = found
         return found
 
+    def _reconcile(self, tablename, idcolumn, label, table, value):
+        """Bind value to the closest existing row when within threshold.
+
+        Returns the matched id and records an event, or None when reconciliation
+        is off, the table is not reconcilable, or nothing is close enough.
+        """
+        if self.reconcile_threshold <= 0 or tablename not in RECONCILABLE_TABLES:
+            return None
+
+        wanted = normalize_label(value)
+        if not wanted:
+            return None
+
+        best_id = None
+        best_label = None
+        best_score = 0.0
+        for existing in db.session.execute(table.select()):
+            candidate = getattr(existing, label, None)
+            normalized = normalize_label(candidate)
+            if not normalized:
+                continue
+            if normalized == wanted:
+                score = 1.0
+            else:
+                score = difflib.SequenceMatcher(None, wanted, normalized).ratio()
+            if score > best_score:
+                best_score = score
+                best_id = getattr(existing, idcolumn)
+                best_label = candidate
+
+        if best_id is not None and best_score >= self.reconcile_threshold:
+            self.events.append({
+                'table': tablename,
+                'input': str(value).strip(),
+                'matched': best_label,
+                'score': round(best_score, 3),
+            })
+            return best_id
+        return None
+
+    def take_events(self):
+        """Return and clear reconciliations recorded since the last call."""
+        events = self.events
+        self.events = []
+        return events
+
     def remember(self, tablename, name, rowid):
         if name is None:
             return
@@ -282,6 +377,7 @@ class ImportResult(object):
         self.updated = 0
         self.problems = []
         self.skipped = 0
+        self.reconciliations = []
 
     @property
     def ok(self):
@@ -381,6 +477,10 @@ def import_csv(tablename, text, resolver=None, commit=False):
                     continue
                 try:
                     values[name] = resolver.resolve(target[0], target[1], text)
+                    for event in resolver.take_events():
+                        event['line'] = line
+                        event['column'] = name
+                        result.reconciliations.append(event)
                 except ValueError as exc:
                     result.problems.append(RowProblem(line, name, str(exc)))
                     failed = True