Release 0.8.0: Windows installer, twelve fixes from a real Server 2019 install, operator docs
This commit is contained in:
500
shopdb/core/services/csvimport.py
Normal file
500
shopdb/core/services/csvimport.py
Normal file
@@ -0,0 +1,500 @@
|
||||
"""Bulk-load a site's starting data from CSV files.
|
||||
|
||||
WHY THIS EXISTS. Adopting a site means getting its existing asset register into
|
||||
ShopDB. The HTTP import surface (the IMPORT-API wiki page) is the right tool when
|
||||
there is a source system to read from and someone able to script against it. A
|
||||
sister site with a spreadsheet and no developer needs something else, and that
|
||||
is the common case.
|
||||
|
||||
THE DESIGN PROBLEM, which a hand-written template set gets wrong. A CSV row for
|
||||
an asset has to say where the asset is. The database stores `locationid`, an
|
||||
integer. Nobody types integers into a spreadsheet correctly, and requiring it
|
||||
means the operator must first import locations, read back the generated ids, and
|
||||
paste them into the asset sheet. That is not a workflow anyone completes.
|
||||
|
||||
So every foreign key here accepts EITHER a numeric id OR the referenced row's
|
||||
name, and names are resolved at import time:
|
||||
|
||||
assetnumber,assettypeid,locationid,businessunitid
|
||||
CMM-01,Measuring Tool,Building 1 Bay 3,Quality
|
||||
|
||||
The column keeps its database name - the convention in CONTRIBUTING.md - while
|
||||
the VALUE is whatever the operator actually knows. A name that does not resolve
|
||||
is a row error naming the column, the value, and where to fix it, not a foreign
|
||||
key constraint violation from three layers down.
|
||||
|
||||
IDEMPOTENCE. Each table declares a natural key (`assetnumber`, `locationname`,
|
||||
...). Re-importing a file updates the matched rows rather than duplicating them,
|
||||
so a site can correct its spreadsheet and run it again - which they will.
|
||||
|
||||
SAFETY. Validation is a separate pass over the whole file BEFORE anything is
|
||||
written, and the default is a dry run. An import either applies completely or
|
||||
not at all.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
|
||||
from sqlalchemy import inspect as sqla_inspect
|
||||
|
||||
from shopdb.extensions import db
|
||||
|
||||
|
||||
class ImportError_(Exception):
|
||||
"""A problem with the request itself, not with a row."""
|
||||
|
||||
|
||||
# The tables a site may load, in dependency order: a table never appears before
|
||||
# something it references. `label` is the human-facing column - the one an
|
||||
# operator would write in a spreadsheet, and the one used to resolve this table
|
||||
# when another table points at it. `natural` is what makes a re-import an update
|
||||
# rather than a duplicate.
|
||||
#
|
||||
# Deliberately NOT every table. Transactional and derived data (audit rows,
|
||||
# reports, enforcement history) has no business arriving by spreadsheet, and
|
||||
# offering it would invite someone to try.
|
||||
IMPORTABLE = OrderedDict([
|
||||
('assetstatuses', {'label': 'status', 'natural': 'status'}),
|
||||
('assettypes', {'label': 'assettype', 'natural': 'assettype'}),
|
||||
('locationtypes', {'label': 'locationtype', 'natural': 'locationtype'}),
|
||||
('modeltypes', {'label': 'modeltype', 'natural': 'modeltype'}),
|
||||
('computertypes', {'label': 'computertype', 'natural': 'computertype'}),
|
||||
('machinetypes', {'label': 'machinetype', 'natural': 'machinetype'}),
|
||||
('businessunits', {'label': 'businessunit', 'natural': 'businessunit'}),
|
||||
('locations', {'label': 'locationname', 'natural': 'locationname'}),
|
||||
('vendors', {'label': 'vendor', 'natural': 'vendor'}),
|
||||
('models', {'label': 'modelnumber', 'natural': 'modelnumber'}),
|
||||
('operatingsystems', {'label': 'osname', 'natural': 'osname'}),
|
||||
('assets', {'label': 'assetnumber', 'natural': 'assetnumber'}),
|
||||
('computers', {'label': 'hostname', 'natural': 'assetid'}),
|
||||
('machines', {'label': None, 'natural': 'assetid'}),
|
||||
])
|
||||
|
||||
# Columns a CSV may never set: surrogate keys the database owns, and audit
|
||||
# stamps. Accepting an id from a spreadsheet is how two sites end up with
|
||||
# colliding primary keys the first time anyone merges data.
|
||||
NEVER_IMPORT = {'createddate', 'modifieddate'}
|
||||
|
||||
# Credentials are never taken from a file. A CSV that carries password hashes
|
||||
# gets mailed around, and one that carries plaintext is worse. Users are created
|
||||
# with a random password and must change it at first login.
|
||||
PASSWORD_COLUMNS = {'passwordhash', 'password'}
|
||||
|
||||
TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'}
|
||||
FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''}
|
||||
|
||||
|
||||
def table_for(name):
|
||||
table = db.metadata.tables.get(name)
|
||||
if table is None:
|
||||
raise ImportError_(
|
||||
"no such table: %s\nImportable: %s" % (name, ', '.join(IMPORTABLE)))
|
||||
return table
|
||||
|
||||
|
||||
def assert_schema_ready(tablename):
|
||||
"""Fail with a sentence, not a traceback, when the schema is not there yet.
|
||||
|
||||
Running this against a database that has never been migrated is an easy
|
||||
mistake - a fresh site, or the wrong DATABASE_URL - and SQLAlchemy's answer
|
||||
is a wall of stack frames ending in 'no such table'. The operator needs to
|
||||
be told to run the migrations.
|
||||
"""
|
||||
# The SESSION's connection, not db.engine. Taking a separate connection ends
|
||||
# up rolling back the session's in-flight transaction under a shared-
|
||||
# connection pool, which silently discarded rows that had just been written.
|
||||
inspector = sqla_inspect(db.session.connection())
|
||||
if not inspector.has_table(tablename):
|
||||
raise ImportError_(
|
||||
"the '%s' table does not exist in this database.\n"
|
||||
" Run 'flask db upgrade' first, and check DATABASE_URL points at the\n"
|
||||
" site you meant." % tablename)
|
||||
|
||||
|
||||
def importable_columns(tablename):
|
||||
"""Columns an operator may set, in a stable order.
|
||||
|
||||
Primary keys are excluded - they are the database's - EXCEPT where the key
|
||||
is also a foreign key, as on the subtype tables where `assetid` both
|
||||
identifies the row and points at its asset.
|
||||
"""
|
||||
table = table_for(tablename)
|
||||
primary = {c.name for c in table.primary_key}
|
||||
out = []
|
||||
for column in table.columns:
|
||||
if column.name in NEVER_IMPORT:
|
||||
continue
|
||||
if column.name in primary and not column.foreign_keys:
|
||||
continue
|
||||
if column.name in PASSWORD_COLUMNS:
|
||||
continue
|
||||
out.append(column)
|
||||
return out
|
||||
|
||||
|
||||
def required_columns(tablename):
|
||||
primary = {c.name for c in table_for(tablename).primary_key}
|
||||
return [
|
||||
c.name for c in importable_columns(tablename)
|
||||
if not c.nullable and c.name not in primary
|
||||
and c.default is None and c.server_default is None
|
||||
]
|
||||
|
||||
|
||||
def _fk_target(column):
|
||||
"""(tablename, idcolumn) this column points at, or None."""
|
||||
if not column.foreign_keys:
|
||||
return None
|
||||
target = list(column.foreign_keys)[0].column
|
||||
return target.table.name, target.name
|
||||
|
||||
|
||||
def _label_column(tablename):
|
||||
"""The human-readable column for a table, used to resolve references to it.
|
||||
|
||||
Derived where possible so this does not need maintaining: the schema names
|
||||
reference tables' label column after the table itself (assettypes.assettype,
|
||||
locationtypes.locationtype). IMPORTABLE overrides where that rule does not
|
||||
hold.
|
||||
"""
|
||||
spec = IMPORTABLE.get(tablename)
|
||||
if spec and spec.get('label'):
|
||||
return spec['label']
|
||||
table = db.metadata.tables.get(tablename)
|
||||
if table is None:
|
||||
return None
|
||||
singular = tablename[:-1] if tablename.endswith('s') else tablename
|
||||
for candidate in (singular, singular + 'name', tablename + 'name'):
|
||||
if candidate in table.columns:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _coerce(column, raw):
|
||||
"""Turn a CSV string into something the column will accept."""
|
||||
text = (raw or '').strip()
|
||||
typename = column.type.__class__.__name__.upper()
|
||||
|
||||
if text == '':
|
||||
return None
|
||||
|
||||
if 'BOOL' in typename:
|
||||
low = text.lower()
|
||||
if low in TRUE_VALUES:
|
||||
return True
|
||||
if low in FALSE_VALUES:
|
||||
return False
|
||||
raise ValueError("expected 1 or 0, got '%s'" % text)
|
||||
|
||||
if 'INT' in typename:
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
raise ValueError("expected a whole number, got '%s'" % text)
|
||||
|
||||
if 'FLOAT' in typename or 'NUMERIC' in typename or 'DECIMAL' in typename:
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
raise ValueError("expected a number, got '%s'" % text)
|
||||
|
||||
if 'DATE' in typename or 'TIME' in typename:
|
||||
from datetime import datetime
|
||||
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'):
|
||||
try:
|
||||
return datetime.strptime(text, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
raise ValueError("expected a date like 2026-08-04, got '%s'" % text)
|
||||
|
||||
length = getattr(column.type, 'length', None)
|
||||
if length and len(text) > length:
|
||||
raise ValueError("longer than the column allows (%d > %d characters)" % (len(text), length))
|
||||
return text
|
||||
|
||||
|
||||
class Resolver:
|
||||
"""Turns names into ids, remembering what it has already looked up.
|
||||
|
||||
Rows created earlier in the SAME import are resolvable, so one file can
|
||||
reference another that was loaded moments ago without a commit in between.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._cache = {}
|
||||
|
||||
def resolve(self, tablename, idcolumn, value):
|
||||
if value is None:
|
||||
return None
|
||||
# A number is taken at face value: exports round-trip, and some sites do
|
||||
# know their ids.
|
||||
if isinstance(value, int) or (isinstance(value, str) and value.strip().isdigit()):
|
||||
return int(value)
|
||||
|
||||
label = _label_column(tablename)
|
||||
if not label:
|
||||
raise ValueError(
|
||||
"cannot look up %s by name - give the numeric %s instead" % (tablename, idcolumn))
|
||||
|
||||
key = (tablename, str(value).strip().lower())
|
||||
if key in self._cache:
|
||||
return self._cache[key]
|
||||
|
||||
table = db.metadata.tables[tablename]
|
||||
row = db.session.execute(
|
||||
table.select().where(
|
||||
db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower())
|
||||
).first()
|
||||
if row is None:
|
||||
raise ValueError(
|
||||
"nothing in %s is named '%s' - add it to %s.csv, or import that file first"
|
||||
% (tablename, value, tablename))
|
||||
found = getattr(row, idcolumn)
|
||||
self._cache[key] = found
|
||||
return found
|
||||
|
||||
def remember(self, tablename, name, rowid):
|
||||
if name is None:
|
||||
return
|
||||
self._cache[(tablename, str(name).strip().lower())] = rowid
|
||||
|
||||
|
||||
class RowProblem(object):
|
||||
def __init__(self, line, column, message):
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.message = message
|
||||
|
||||
def __str__(self):
|
||||
where = "line %d" % self.line
|
||||
if self.column:
|
||||
where += ", column '%s'" % self.column
|
||||
return "%s: %s" % (where, self.message)
|
||||
|
||||
|
||||
class ImportResult(object):
|
||||
def __init__(self, tablename):
|
||||
self.tablename = tablename
|
||||
self.created = 0
|
||||
self.updated = 0
|
||||
self.problems = []
|
||||
self.skipped = 0
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
return not self.problems
|
||||
|
||||
def summary(self):
|
||||
return "%s: %d new, %d updated, %d problem(s)" % (
|
||||
self.tablename, self.created, self.updated, len(self.problems))
|
||||
|
||||
|
||||
def read_rows(text):
|
||||
"""Rows as dicts, with the sample/comment lines the templates carry removed."""
|
||||
reader = csv.reader(io.StringIO(text))
|
||||
header = None
|
||||
rows = []
|
||||
for number, fields in enumerate(reader, 1):
|
||||
if not fields or all(not f.strip() for f in fields):
|
||||
continue
|
||||
if fields[0].lstrip().startswith('#'):
|
||||
continue
|
||||
if header is None:
|
||||
# Strip a UTF-8 BOM if one survived: Excel writes it, and a header
|
||||
# named '\ufeffassetnumber' would match nothing.
|
||||
header = [f.strip().lstrip('\ufeff') for f in fields]
|
||||
continue
|
||||
# A short row is padded rather than rejected: trailing empty cells are
|
||||
# routinely dropped by spreadsheet exports.
|
||||
padded = list(fields) + [''] * (len(header) - len(fields))
|
||||
rows.append((number, dict(zip(header, padded))))
|
||||
if header is None:
|
||||
raise ImportError_('the file has no header row')
|
||||
return header, rows
|
||||
|
||||
|
||||
def import_csv(tablename, text, resolver=None, commit=False):
|
||||
"""Validate a CSV against one table and write it into the current transaction.
|
||||
|
||||
Validation runs over EVERY row first. Nothing is written unless the whole
|
||||
file is clean, so a typo on line 400 does not leave 399 rows imported.
|
||||
|
||||
Writes always go into the SESSION; this never commits. The caller commits or
|
||||
rolls back, which is what makes a dry run over a whole folder work: assets
|
||||
can resolve the locations the previous file just created, because they are
|
||||
really there inside the transaction, and the rollback removes them again.
|
||||
Skipping the writes on a dry run instead made every cross-file reference
|
||||
fail, which is the one thing a folder-wide check exists to verify.
|
||||
|
||||
`commit` is retained only for callers that want the old explicit read.
|
||||
"""
|
||||
table = table_for(tablename)
|
||||
spec = IMPORTABLE.get(tablename)
|
||||
if spec is None:
|
||||
raise ImportError_(
|
||||
"%s is not importable.\nImportable: %s" % (tablename, ', '.join(IMPORTABLE)))
|
||||
|
||||
assert_schema_ready(tablename)
|
||||
resolver = resolver or Resolver()
|
||||
result = ImportResult(tablename)
|
||||
header, rows = read_rows(text)
|
||||
|
||||
allowed = {c.name: c for c in importable_columns(tablename)}
|
||||
unknown = [h for h in header if h not in allowed]
|
||||
if unknown:
|
||||
known = ', '.join(sorted(allowed))
|
||||
result.problems.append(RowProblem(
|
||||
1, None, "unknown column(s): %s\n this table accepts: %s" % (', '.join(unknown), known)))
|
||||
return result
|
||||
|
||||
missing = [c for c in required_columns(tablename) if c not in header]
|
||||
if missing:
|
||||
result.problems.append(RowProblem(
|
||||
1, None, "required column(s) missing: %s" % ', '.join(missing)))
|
||||
return result
|
||||
|
||||
natural = spec['natural']
|
||||
if natural not in header:
|
||||
result.problems.append(RowProblem(
|
||||
1, None, "the '%s' column identifies each row and must be present" % natural))
|
||||
return result
|
||||
|
||||
prepared = []
|
||||
for line, raw in rows:
|
||||
values = {}
|
||||
failed = False
|
||||
for name, column in allowed.items():
|
||||
if name not in raw:
|
||||
continue
|
||||
target = _fk_target(column)
|
||||
# RESOLVE BEFORE COERCING. A foreign key column is an integer, so
|
||||
# coercing first rejects every name with "expected a whole number" -
|
||||
# which defeats the entire point of accepting names. Resolution
|
||||
# handles the numeric case itself.
|
||||
if target:
|
||||
text = (raw[name] or '').strip()
|
||||
if text == '':
|
||||
values[name] = None
|
||||
continue
|
||||
try:
|
||||
values[name] = resolver.resolve(target[0], target[1], text)
|
||||
except ValueError as exc:
|
||||
result.problems.append(RowProblem(line, name, str(exc)))
|
||||
failed = True
|
||||
continue
|
||||
try:
|
||||
values[name] = _coerce(column, raw[name])
|
||||
except ValueError as exc:
|
||||
result.problems.append(RowProblem(line, name, str(exc)))
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
continue
|
||||
if values.get(natural) in (None, ''):
|
||||
result.problems.append(RowProblem(line, natural, 'must have a value'))
|
||||
continue
|
||||
prepared.append((line, values))
|
||||
|
||||
if not result.ok:
|
||||
return result
|
||||
|
||||
# Second pass: decide insert vs update. Counted even on a dry run, because
|
||||
# "this will change 4000 rows" is the number an operator needs before saying
|
||||
# yes.
|
||||
label = _label_column(tablename)
|
||||
for line, values in prepared:
|
||||
existing = db.session.execute(
|
||||
table.select().where(table.c[natural] == values[natural])).first()
|
||||
if existing is None:
|
||||
result.created += 1
|
||||
inserted = db.session.execute(table.insert().values(**values))
|
||||
newid = inserted.inserted_primary_key[0] if inserted.inserted_primary_key else None
|
||||
if label and label in values:
|
||||
resolver.remember(tablename, values[label], newid)
|
||||
else:
|
||||
result.updated += 1
|
||||
db.session.execute(
|
||||
table.update().where(table.c[natural] == values[natural]).values(**values))
|
||||
if label and label in values:
|
||||
primary = list(table.primary_key)[0].name
|
||||
resolver.remember(tablename, values[label], getattr(existing, primary))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def generate_template(tablename):
|
||||
"""A header row plus a commented example, built from the live schema.
|
||||
|
||||
Generated rather than maintained: a hand-written template set drifts on the
|
||||
next migration, and silently - the file still looks right.
|
||||
"""
|
||||
columns = importable_columns(tablename)
|
||||
required = set(required_columns(tablename))
|
||||
header = [c.name for c in columns]
|
||||
|
||||
notes = []
|
||||
for column in columns:
|
||||
bits = []
|
||||
bits.append('REQUIRED' if column.name in required else 'optional')
|
||||
target = _fk_target(column)
|
||||
if target:
|
||||
label = _label_column(target[0])
|
||||
if label:
|
||||
bits.append("name from %s.csv (or a numeric id)" % target[0])
|
||||
else:
|
||||
bits.append("numeric %s id" % target[0])
|
||||
else:
|
||||
typename = column.type.__class__.__name__.upper()
|
||||
if 'BOOL' in typename:
|
||||
bits.append('1 or 0')
|
||||
elif 'DATE' in typename or 'TIME' in typename:
|
||||
bits.append('YYYY-MM-DD')
|
||||
elif getattr(column.type, 'length', None):
|
||||
bits.append('text, max %d' % column.type.length)
|
||||
notes.append("# %-24s %s" % (column.name, ', '.join(bits)))
|
||||
|
||||
spec = IMPORTABLE[tablename]
|
||||
lines = [
|
||||
"# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename,
|
||||
"#",
|
||||
"# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'],
|
||||
"# than creating a second one, so it is safe to correct this file and run again.",
|
||||
"#",
|
||||
"# Columns:",
|
||||
] + notes + [
|
||||
"#",
|
||||
"# Lines starting with # are ignored. Delete the example row before importing.",
|
||||
','.join(header),
|
||||
'# ' + ','.join(_example(c, required) for c in columns),
|
||||
]
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def _example(column, required):
|
||||
target = _fk_target(column)
|
||||
if target:
|
||||
label = _label_column(target[0])
|
||||
return ('<%s name>' % target[0][:-1]) if label else '<%s id>' % target[0]
|
||||
typename = column.type.__class__.__name__.upper()
|
||||
if 'BOOL' in typename:
|
||||
return '1'
|
||||
if 'DATE' in typename or 'TIME' in typename:
|
||||
return '2026-08-04'
|
||||
if 'INT' in typename or 'FLOAT' in typename or 'NUMERIC' in typename:
|
||||
return '0'
|
||||
return '<%s>' % column.name
|
||||
|
||||
|
||||
def dependency_order(names):
|
||||
"""The given tables, ordered so a table never precedes what it references."""
|
||||
known = [t for t in IMPORTABLE if t in set(names)]
|
||||
return known
|
||||
|
||||
|
||||
def table_from_filename(filename):
|
||||
stem = re.sub(r'\.csv$', '', filename.strip(), flags=re.I)
|
||||
stem = stem.rsplit('/', 1)[-1].rsplit('\\', 1)[-1].lower()
|
||||
return stem
|
||||
Reference in New Issue
Block a user