Release 0.8.0: Windows installer, twelve fixes from a real Server 2019 install, operator docs
This commit is contained in:
@@ -42,7 +42,7 @@ __contract_version__ = '0.15.0'
|
||||
# plugin-contract version above are distinct series with independent
|
||||
# bump rules. Not part of the shopdb.api contract surface, so it is
|
||||
# not re-exported there.
|
||||
__version__ = '0.7.0'
|
||||
__version__ = '0.8.0'
|
||||
|
||||
|
||||
def create_app(config_name: str = None) -> Flask:
|
||||
@@ -172,12 +172,13 @@ def register_blueprints(app: Flask):
|
||||
def register_cli_commands(app: Flask):
|
||||
"""Register Flask CLI commands."""
|
||||
from .plugins.cli import plugin_cli
|
||||
from .cli import db_cli, seed_cli, relationships_cli
|
||||
from .cli import db_cli, seed_cli, relationships_cli, csv_cli
|
||||
|
||||
app.cli.add_command(plugin_cli)
|
||||
app.cli.add_command(db_cli)
|
||||
app.cli.add_command(seed_cli)
|
||||
app.cli.add_command(relationships_cli)
|
||||
app.cli.add_command(csv_cli)
|
||||
|
||||
|
||||
def register_error_handlers(app: Flask):
|
||||
|
||||
@@ -825,3 +825,160 @@ def seed_demo_clear(yes):
|
||||
click.echo(click.style(
|
||||
f"Removed {len(demo_ids)} demo assets and "
|
||||
f"{parts_deleted} sample parts.", fg='green'))
|
||||
|
||||
|
||||
@click.group('csv')
|
||||
def csv_cli():
|
||||
"""Load a site's starting data from CSV files."""
|
||||
pass
|
||||
|
||||
|
||||
@csv_cli.command('templates')
|
||||
@click.option('--out', 'outdir', default='csv-templates',
|
||||
help='Directory to write the templates into.')
|
||||
@with_appcontext
|
||||
def csv_templates(outdir):
|
||||
"""Write a CSV template per importable table, generated from the schema.
|
||||
|
||||
Generated rather than kept by hand: a maintained template set drifts on the
|
||||
next migration and does so silently, since the file still looks correct.
|
||||
"""
|
||||
import os
|
||||
from shopdb.core.services.csvimport import IMPORTABLE, generate_template
|
||||
|
||||
if not os.path.isdir(outdir):
|
||||
os.makedirs(outdir)
|
||||
for tablename in IMPORTABLE:
|
||||
path = os.path.join(outdir, tablename + '.csv')
|
||||
with open(path, 'w', encoding='utf-8') as handle:
|
||||
handle.write(generate_template(tablename))
|
||||
click.echo(' %s' % path)
|
||||
|
||||
readme = os.path.join(outdir, 'README.txt')
|
||||
with open(readme, 'w', encoding='utf-8') as handle:
|
||||
handle.write(
|
||||
'ShopDB-Flask import templates\n'
|
||||
'=============================\n\n'
|
||||
'Generated from the live database schema. Every column here exists;\n'
|
||||
'every required column is marked.\n\n'
|
||||
'Fill in the ones you need - you do not need all of them.\n\n'
|
||||
'Foreign keys take a NAME or a numeric id. Write the name:\n'
|
||||
' locationid -> Building 1 Bay 3\n'
|
||||
' vendorid -> Haas Automation\n'
|
||||
'The importer resolves it, and tells you which row and column to fix\n'
|
||||
'if the name is not found.\n\n'
|
||||
'Import the whole folder at once and order is handled for you:\n\n'
|
||||
' flask csv import --dir . (checks only, changes nothing)\n'
|
||||
' flask csv import --dir . --commit (applies)\n\n'
|
||||
'Nothing is written unless every row passes, so a mistake on line 400\n'
|
||||
'does not leave 399 rows half-imported.\n\n'
|
||||
'User accounts are deliberately not importable here: passwords do not\n'
|
||||
'belong in a spreadsheet.\n')
|
||||
click.echo(' %s' % readme)
|
||||
click.echo('')
|
||||
click.echo(click.style('%d templates written to %s' % (len(IMPORTABLE), outdir),
|
||||
fg='green', bold=True))
|
||||
|
||||
|
||||
@csv_cli.command('import')
|
||||
@click.option('--file', 'path', default=None, help='One CSV file.')
|
||||
@click.option('--dir', 'directory', default=None,
|
||||
help='A folder of CSVs, imported in dependency order.')
|
||||
@click.option('--table', 'tablename', default=None,
|
||||
help='Target table. Defaults to the file name.')
|
||||
@click.option('--commit', is_flag=True, default=False,
|
||||
help='Apply the changes. Without this, nothing is written.')
|
||||
@with_appcontext
|
||||
def csv_import(path, directory, tablename, commit):
|
||||
"""Validate CSVs and, with --commit, load them.
|
||||
|
||||
Dry run by default. The report is the same either way, so what you review is
|
||||
what you get.
|
||||
"""
|
||||
import os
|
||||
from shopdb.extensions import db
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from shopdb.core.services.csvimport import (
|
||||
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
|
||||
|
||||
if not path and not directory:
|
||||
raise click.UsageError('give --file or --dir')
|
||||
|
||||
jobs = []
|
||||
if path:
|
||||
jobs.append((tablename or table_from_filename(os.path.basename(path)), path))
|
||||
else:
|
||||
found = {}
|
||||
for name in os.listdir(directory):
|
||||
if not name.lower().endswith('.csv'):
|
||||
continue
|
||||
found[table_from_filename(name)] = os.path.join(directory, name)
|
||||
ordered = dependency_order(list(found))
|
||||
unknown = sorted(set(found) - set(ordered))
|
||||
for name in ordered:
|
||||
jobs.append((name, found[name]))
|
||||
if unknown:
|
||||
click.echo(click.style(
|
||||
'skipping (not importable): %s' % ', '.join(unknown), fg='yellow'))
|
||||
|
||||
if not jobs:
|
||||
click.echo('nothing to do - no CSV files found')
|
||||
return
|
||||
|
||||
click.echo(click.style(
|
||||
'Checking %d file(s)%s' % (len(jobs), '' if commit else ' - DRY RUN, nothing will be written'),
|
||||
bold=True))
|
||||
click.echo('')
|
||||
|
||||
resolver = Resolver()
|
||||
results = []
|
||||
failed = False
|
||||
for name, filepath in jobs:
|
||||
with open(filepath, 'r', encoding='utf-8-sig') as handle:
|
||||
text = handle.read()
|
||||
try:
|
||||
# Every file is applied inside ONE transaction, so a failure part way
|
||||
# through a folder rolls the whole run back rather than leaving the
|
||||
# site half-populated.
|
||||
result = import_csv(name, text, resolver=resolver, commit=commit)
|
||||
except ImportError_ as exc:
|
||||
click.echo(click.style(' %-18s %s' % (name, exc), fg='red'))
|
||||
failed = True
|
||||
continue
|
||||
except SQLAlchemyError as exc:
|
||||
# Anything the database itself refuses. The operator gets the cause
|
||||
# in one line rather than a traceback they cannot act on.
|
||||
db.session.rollback()
|
||||
click.echo(click.style(' %-18s database error: %s'
|
||||
% (name, str(exc).split(chr(10))[0]), fg='red'))
|
||||
failed = True
|
||||
continue
|
||||
results.append(result)
|
||||
colour = 'green' if result.ok else 'red'
|
||||
click.echo(click.style(' ' + result.summary(), fg=colour))
|
||||
for problem in result.problems[:20]:
|
||||
click.echo(click.style(' %s' % problem, fg='red'))
|
||||
if len(result.problems) > 20:
|
||||
click.echo(click.style(' ... and %d more' % (len(result.problems) - 20), fg='red'))
|
||||
if not result.ok:
|
||||
failed = True
|
||||
|
||||
click.echo('')
|
||||
if failed:
|
||||
db.session.rollback()
|
||||
click.echo(click.style('Nothing was imported. Fix the problems above and run again.',
|
||||
fg='red', bold=True))
|
||||
raise SystemExit(1)
|
||||
|
||||
total_new = sum(r.created for r in results)
|
||||
total_upd = sum(r.updated for r in results)
|
||||
if commit:
|
||||
db.session.commit()
|
||||
click.echo(click.style('Imported: %d new, %d updated.' % (total_new, total_upd),
|
||||
fg='green', bold=True))
|
||||
else:
|
||||
db.session.rollback()
|
||||
click.echo(click.style(
|
||||
'Looks good: %d would be created, %d updated.' % (total_new, total_upd),
|
||||
fg='green', bold=True))
|
||||
click.echo('Run again with --commit to apply.')
|
||||
|
||||
500
shopdb/core/services/csvimport.py
Normal file
500
shopdb/core/services/csvimport.py
Normal file
@@ -0,0 +1,500 @@
|
||||
"""Bulk-load a site's starting data from CSV files.
|
||||
|
||||
WHY THIS EXISTS. Adopting a site means getting its existing asset register into
|
||||
ShopDB. The HTTP import surface (the IMPORT-API wiki page) is the right tool when
|
||||
there is a source system to read from and someone able to script against it. A
|
||||
sister site with a spreadsheet and no developer needs something else, and that
|
||||
is the common case.
|
||||
|
||||
THE DESIGN PROBLEM, which a hand-written template set gets wrong. A CSV row for
|
||||
an asset has to say where the asset is. The database stores `locationid`, an
|
||||
integer. Nobody types integers into a spreadsheet correctly, and requiring it
|
||||
means the operator must first import locations, read back the generated ids, and
|
||||
paste them into the asset sheet. That is not a workflow anyone completes.
|
||||
|
||||
So every foreign key here accepts EITHER a numeric id OR the referenced row's
|
||||
name, and names are resolved at import time:
|
||||
|
||||
assetnumber,assettypeid,locationid,businessunitid
|
||||
CMM-01,Measuring Tool,Building 1 Bay 3,Quality
|
||||
|
||||
The column keeps its database name - the convention in CONTRIBUTING.md - while
|
||||
the VALUE is whatever the operator actually knows. A name that does not resolve
|
||||
is a row error naming the column, the value, and where to fix it, not a foreign
|
||||
key constraint violation from three layers down.
|
||||
|
||||
IDEMPOTENCE. Each table declares a natural key (`assetnumber`, `locationname`,
|
||||
...). Re-importing a file updates the matched rows rather than duplicating them,
|
||||
so a site can correct its spreadsheet and run it again - which they will.
|
||||
|
||||
SAFETY. Validation is a separate pass over the whole file BEFORE anything is
|
||||
written, and the default is a dry run. An import either applies completely or
|
||||
not at all.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
|
||||
from sqlalchemy import inspect as sqla_inspect
|
||||
|
||||
from shopdb.extensions import db
|
||||
|
||||
|
||||
class ImportError_(Exception):
|
||||
"""A problem with the request itself, not with a row."""
|
||||
|
||||
|
||||
# The tables a site may load, in dependency order: a table never appears before
|
||||
# something it references. `label` is the human-facing column - the one an
|
||||
# operator would write in a spreadsheet, and the one used to resolve this table
|
||||
# when another table points at it. `natural` is what makes a re-import an update
|
||||
# rather than a duplicate.
|
||||
#
|
||||
# Deliberately NOT every table. Transactional and derived data (audit rows,
|
||||
# reports, enforcement history) has no business arriving by spreadsheet, and
|
||||
# offering it would invite someone to try.
|
||||
IMPORTABLE = OrderedDict([
|
||||
('assetstatuses', {'label': 'status', 'natural': 'status'}),
|
||||
('assettypes', {'label': 'assettype', 'natural': 'assettype'}),
|
||||
('locationtypes', {'label': 'locationtype', 'natural': 'locationtype'}),
|
||||
('modeltypes', {'label': 'modeltype', 'natural': 'modeltype'}),
|
||||
('computertypes', {'label': 'computertype', 'natural': 'computertype'}),
|
||||
('machinetypes', {'label': 'machinetype', 'natural': 'machinetype'}),
|
||||
('businessunits', {'label': 'businessunit', 'natural': 'businessunit'}),
|
||||
('locations', {'label': 'locationname', 'natural': 'locationname'}),
|
||||
('vendors', {'label': 'vendor', 'natural': 'vendor'}),
|
||||
('models', {'label': 'modelnumber', 'natural': 'modelnumber'}),
|
||||
('operatingsystems', {'label': 'osname', 'natural': 'osname'}),
|
||||
('assets', {'label': 'assetnumber', 'natural': 'assetnumber'}),
|
||||
('computers', {'label': 'hostname', 'natural': 'assetid'}),
|
||||
('machines', {'label': None, 'natural': 'assetid'}),
|
||||
])
|
||||
|
||||
# Columns a CSV may never set: surrogate keys the database owns, and audit
|
||||
# stamps. Accepting an id from a spreadsheet is how two sites end up with
|
||||
# colliding primary keys the first time anyone merges data.
|
||||
NEVER_IMPORT = {'createddate', 'modifieddate'}
|
||||
|
||||
# Credentials are never taken from a file. A CSV that carries password hashes
|
||||
# gets mailed around, and one that carries plaintext is worse. Users are created
|
||||
# with a random password and must change it at first login.
|
||||
PASSWORD_COLUMNS = {'passwordhash', 'password'}
|
||||
|
||||
TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'}
|
||||
FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''}
|
||||
|
||||
|
||||
def table_for(name):
|
||||
table = db.metadata.tables.get(name)
|
||||
if table is None:
|
||||
raise ImportError_(
|
||||
"no such table: %s\nImportable: %s" % (name, ', '.join(IMPORTABLE)))
|
||||
return table
|
||||
|
||||
|
||||
def assert_schema_ready(tablename):
|
||||
"""Fail with a sentence, not a traceback, when the schema is not there yet.
|
||||
|
||||
Running this against a database that has never been migrated is an easy
|
||||
mistake - a fresh site, or the wrong DATABASE_URL - and SQLAlchemy's answer
|
||||
is a wall of stack frames ending in 'no such table'. The operator needs to
|
||||
be told to run the migrations.
|
||||
"""
|
||||
# The SESSION's connection, not db.engine. Taking a separate connection ends
|
||||
# up rolling back the session's in-flight transaction under a shared-
|
||||
# connection pool, which silently discarded rows that had just been written.
|
||||
inspector = sqla_inspect(db.session.connection())
|
||||
if not inspector.has_table(tablename):
|
||||
raise ImportError_(
|
||||
"the '%s' table does not exist in this database.\n"
|
||||
" Run 'flask db upgrade' first, and check DATABASE_URL points at the\n"
|
||||
" site you meant." % tablename)
|
||||
|
||||
|
||||
def importable_columns(tablename):
|
||||
"""Columns an operator may set, in a stable order.
|
||||
|
||||
Primary keys are excluded - they are the database's - EXCEPT where the key
|
||||
is also a foreign key, as on the subtype tables where `assetid` both
|
||||
identifies the row and points at its asset.
|
||||
"""
|
||||
table = table_for(tablename)
|
||||
primary = {c.name for c in table.primary_key}
|
||||
out = []
|
||||
for column in table.columns:
|
||||
if column.name in NEVER_IMPORT:
|
||||
continue
|
||||
if column.name in primary and not column.foreign_keys:
|
||||
continue
|
||||
if column.name in PASSWORD_COLUMNS:
|
||||
continue
|
||||
out.append(column)
|
||||
return out
|
||||
|
||||
|
||||
def required_columns(tablename):
|
||||
primary = {c.name for c in table_for(tablename).primary_key}
|
||||
return [
|
||||
c.name for c in importable_columns(tablename)
|
||||
if not c.nullable and c.name not in primary
|
||||
and c.default is None and c.server_default is None
|
||||
]
|
||||
|
||||
|
||||
def _fk_target(column):
|
||||
"""(tablename, idcolumn) this column points at, or None."""
|
||||
if not column.foreign_keys:
|
||||
return None
|
||||
target = list(column.foreign_keys)[0].column
|
||||
return target.table.name, target.name
|
||||
|
||||
|
||||
def _label_column(tablename):
|
||||
"""The human-readable column for a table, used to resolve references to it.
|
||||
|
||||
Derived where possible so this does not need maintaining: the schema names
|
||||
reference tables' label column after the table itself (assettypes.assettype,
|
||||
locationtypes.locationtype). IMPORTABLE overrides where that rule does not
|
||||
hold.
|
||||
"""
|
||||
spec = IMPORTABLE.get(tablename)
|
||||
if spec and spec.get('label'):
|
||||
return spec['label']
|
||||
table = db.metadata.tables.get(tablename)
|
||||
if table is None:
|
||||
return None
|
||||
singular = tablename[:-1] if tablename.endswith('s') else tablename
|
||||
for candidate in (singular, singular + 'name', tablename + 'name'):
|
||||
if candidate in table.columns:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _coerce(column, raw):
|
||||
"""Turn a CSV string into something the column will accept."""
|
||||
text = (raw or '').strip()
|
||||
typename = column.type.__class__.__name__.upper()
|
||||
|
||||
if text == '':
|
||||
return None
|
||||
|
||||
if 'BOOL' in typename:
|
||||
low = text.lower()
|
||||
if low in TRUE_VALUES:
|
||||
return True
|
||||
if low in FALSE_VALUES:
|
||||
return False
|
||||
raise ValueError("expected 1 or 0, got '%s'" % text)
|
||||
|
||||
if 'INT' in typename:
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
raise ValueError("expected a whole number, got '%s'" % text)
|
||||
|
||||
if 'FLOAT' in typename or 'NUMERIC' in typename or 'DECIMAL' in typename:
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
raise ValueError("expected a number, got '%s'" % text)
|
||||
|
||||
if 'DATE' in typename or 'TIME' in typename:
|
||||
from datetime import datetime
|
||||
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'):
|
||||
try:
|
||||
return datetime.strptime(text, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
raise ValueError("expected a date like 2026-08-04, got '%s'" % text)
|
||||
|
||||
length = getattr(column.type, 'length', None)
|
||||
if length and len(text) > length:
|
||||
raise ValueError("longer than the column allows (%d > %d characters)" % (len(text), length))
|
||||
return text
|
||||
|
||||
|
||||
class Resolver:
|
||||
"""Turns names into ids, remembering what it has already looked up.
|
||||
|
||||
Rows created earlier in the SAME import are resolvable, so one file can
|
||||
reference another that was loaded moments ago without a commit in between.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._cache = {}
|
||||
|
||||
def resolve(self, tablename, idcolumn, value):
|
||||
if value is None:
|
||||
return None
|
||||
# A number is taken at face value: exports round-trip, and some sites do
|
||||
# know their ids.
|
||||
if isinstance(value, int) or (isinstance(value, str) and value.strip().isdigit()):
|
||||
return int(value)
|
||||
|
||||
label = _label_column(tablename)
|
||||
if not label:
|
||||
raise ValueError(
|
||||
"cannot look up %s by name - give the numeric %s instead" % (tablename, idcolumn))
|
||||
|
||||
key = (tablename, str(value).strip().lower())
|
||||
if key in self._cache:
|
||||
return self._cache[key]
|
||||
|
||||
table = db.metadata.tables[tablename]
|
||||
row = db.session.execute(
|
||||
table.select().where(
|
||||
db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower())
|
||||
).first()
|
||||
if row is None:
|
||||
raise ValueError(
|
||||
"nothing in %s is named '%s' - add it to %s.csv, or import that file first"
|
||||
% (tablename, value, tablename))
|
||||
found = getattr(row, idcolumn)
|
||||
self._cache[key] = found
|
||||
return found
|
||||
|
||||
def remember(self, tablename, name, rowid):
|
||||
if name is None:
|
||||
return
|
||||
self._cache[(tablename, str(name).strip().lower())] = rowid
|
||||
|
||||
|
||||
class RowProblem(object):
|
||||
def __init__(self, line, column, message):
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.message = message
|
||||
|
||||
def __str__(self):
|
||||
where = "line %d" % self.line
|
||||
if self.column:
|
||||
where += ", column '%s'" % self.column
|
||||
return "%s: %s" % (where, self.message)
|
||||
|
||||
|
||||
class ImportResult(object):
|
||||
def __init__(self, tablename):
|
||||
self.tablename = tablename
|
||||
self.created = 0
|
||||
self.updated = 0
|
||||
self.problems = []
|
||||
self.skipped = 0
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
return not self.problems
|
||||
|
||||
def summary(self):
|
||||
return "%s: %d new, %d updated, %d problem(s)" % (
|
||||
self.tablename, self.created, self.updated, len(self.problems))
|
||||
|
||||
|
||||
def read_rows(text):
|
||||
"""Rows as dicts, with the sample/comment lines the templates carry removed."""
|
||||
reader = csv.reader(io.StringIO(text))
|
||||
header = None
|
||||
rows = []
|
||||
for number, fields in enumerate(reader, 1):
|
||||
if not fields or all(not f.strip() for f in fields):
|
||||
continue
|
||||
if fields[0].lstrip().startswith('#'):
|
||||
continue
|
||||
if header is None:
|
||||
# Strip a UTF-8 BOM if one survived: Excel writes it, and a header
|
||||
# named '\ufeffassetnumber' would match nothing.
|
||||
header = [f.strip().lstrip('\ufeff') for f in fields]
|
||||
continue
|
||||
# A short row is padded rather than rejected: trailing empty cells are
|
||||
# routinely dropped by spreadsheet exports.
|
||||
padded = list(fields) + [''] * (len(header) - len(fields))
|
||||
rows.append((number, dict(zip(header, padded))))
|
||||
if header is None:
|
||||
raise ImportError_('the file has no header row')
|
||||
return header, rows
|
||||
|
||||
|
||||
def import_csv(tablename, text, resolver=None, commit=False):
|
||||
"""Validate a CSV against one table and write it into the current transaction.
|
||||
|
||||
Validation runs over EVERY row first. Nothing is written unless the whole
|
||||
file is clean, so a typo on line 400 does not leave 399 rows imported.
|
||||
|
||||
Writes always go into the SESSION; this never commits. The caller commits or
|
||||
rolls back, which is what makes a dry run over a whole folder work: assets
|
||||
can resolve the locations the previous file just created, because they are
|
||||
really there inside the transaction, and the rollback removes them again.
|
||||
Skipping the writes on a dry run instead made every cross-file reference
|
||||
fail, which is the one thing a folder-wide check exists to verify.
|
||||
|
||||
`commit` is retained only for callers that want the old explicit read.
|
||||
"""
|
||||
table = table_for(tablename)
|
||||
spec = IMPORTABLE.get(tablename)
|
||||
if spec is None:
|
||||
raise ImportError_(
|
||||
"%s is not importable.\nImportable: %s" % (tablename, ', '.join(IMPORTABLE)))
|
||||
|
||||
assert_schema_ready(tablename)
|
||||
resolver = resolver or Resolver()
|
||||
result = ImportResult(tablename)
|
||||
header, rows = read_rows(text)
|
||||
|
||||
allowed = {c.name: c for c in importable_columns(tablename)}
|
||||
unknown = [h for h in header if h not in allowed]
|
||||
if unknown:
|
||||
known = ', '.join(sorted(allowed))
|
||||
result.problems.append(RowProblem(
|
||||
1, None, "unknown column(s): %s\n this table accepts: %s" % (', '.join(unknown), known)))
|
||||
return result
|
||||
|
||||
missing = [c for c in required_columns(tablename) if c not in header]
|
||||
if missing:
|
||||
result.problems.append(RowProblem(
|
||||
1, None, "required column(s) missing: %s" % ', '.join(missing)))
|
||||
return result
|
||||
|
||||
natural = spec['natural']
|
||||
if natural not in header:
|
||||
result.problems.append(RowProblem(
|
||||
1, None, "the '%s' column identifies each row and must be present" % natural))
|
||||
return result
|
||||
|
||||
prepared = []
|
||||
for line, raw in rows:
|
||||
values = {}
|
||||
failed = False
|
||||
for name, column in allowed.items():
|
||||
if name not in raw:
|
||||
continue
|
||||
target = _fk_target(column)
|
||||
# RESOLVE BEFORE COERCING. A foreign key column is an integer, so
|
||||
# coercing first rejects every name with "expected a whole number" -
|
||||
# which defeats the entire point of accepting names. Resolution
|
||||
# handles the numeric case itself.
|
||||
if target:
|
||||
text = (raw[name] or '').strip()
|
||||
if text == '':
|
||||
values[name] = None
|
||||
continue
|
||||
try:
|
||||
values[name] = resolver.resolve(target[0], target[1], text)
|
||||
except ValueError as exc:
|
||||
result.problems.append(RowProblem(line, name, str(exc)))
|
||||
failed = True
|
||||
continue
|
||||
try:
|
||||
values[name] = _coerce(column, raw[name])
|
||||
except ValueError as exc:
|
||||
result.problems.append(RowProblem(line, name, str(exc)))
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
continue
|
||||
if values.get(natural) in (None, ''):
|
||||
result.problems.append(RowProblem(line, natural, 'must have a value'))
|
||||
continue
|
||||
prepared.append((line, values))
|
||||
|
||||
if not result.ok:
|
||||
return result
|
||||
|
||||
# Second pass: decide insert vs update. Counted even on a dry run, because
|
||||
# "this will change 4000 rows" is the number an operator needs before saying
|
||||
# yes.
|
||||
label = _label_column(tablename)
|
||||
for line, values in prepared:
|
||||
existing = db.session.execute(
|
||||
table.select().where(table.c[natural] == values[natural])).first()
|
||||
if existing is None:
|
||||
result.created += 1
|
||||
inserted = db.session.execute(table.insert().values(**values))
|
||||
newid = inserted.inserted_primary_key[0] if inserted.inserted_primary_key else None
|
||||
if label and label in values:
|
||||
resolver.remember(tablename, values[label], newid)
|
||||
else:
|
||||
result.updated += 1
|
||||
db.session.execute(
|
||||
table.update().where(table.c[natural] == values[natural]).values(**values))
|
||||
if label and label in values:
|
||||
primary = list(table.primary_key)[0].name
|
||||
resolver.remember(tablename, values[label], getattr(existing, primary))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def generate_template(tablename):
|
||||
"""A header row plus a commented example, built from the live schema.
|
||||
|
||||
Generated rather than maintained: a hand-written template set drifts on the
|
||||
next migration, and silently - the file still looks right.
|
||||
"""
|
||||
columns = importable_columns(tablename)
|
||||
required = set(required_columns(tablename))
|
||||
header = [c.name for c in columns]
|
||||
|
||||
notes = []
|
||||
for column in columns:
|
||||
bits = []
|
||||
bits.append('REQUIRED' if column.name in required else 'optional')
|
||||
target = _fk_target(column)
|
||||
if target:
|
||||
label = _label_column(target[0])
|
||||
if label:
|
||||
bits.append("name from %s.csv (or a numeric id)" % target[0])
|
||||
else:
|
||||
bits.append("numeric %s id" % target[0])
|
||||
else:
|
||||
typename = column.type.__class__.__name__.upper()
|
||||
if 'BOOL' in typename:
|
||||
bits.append('1 or 0')
|
||||
elif 'DATE' in typename or 'TIME' in typename:
|
||||
bits.append('YYYY-MM-DD')
|
||||
elif getattr(column.type, 'length', None):
|
||||
bits.append('text, max %d' % column.type.length)
|
||||
notes.append("# %-24s %s" % (column.name, ', '.join(bits)))
|
||||
|
||||
spec = IMPORTABLE[tablename]
|
||||
lines = [
|
||||
"# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename,
|
||||
"#",
|
||||
"# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'],
|
||||
"# than creating a second one, so it is safe to correct this file and run again.",
|
||||
"#",
|
||||
"# Columns:",
|
||||
] + notes + [
|
||||
"#",
|
||||
"# Lines starting with # are ignored. Delete the example row before importing.",
|
||||
','.join(header),
|
||||
'# ' + ','.join(_example(c, required) for c in columns),
|
||||
]
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def _example(column, required):
|
||||
target = _fk_target(column)
|
||||
if target:
|
||||
label = _label_column(target[0])
|
||||
return ('<%s name>' % target[0][:-1]) if label else '<%s id>' % target[0]
|
||||
typename = column.type.__class__.__name__.upper()
|
||||
if 'BOOL' in typename:
|
||||
return '1'
|
||||
if 'DATE' in typename or 'TIME' in typename:
|
||||
return '2026-08-04'
|
||||
if 'INT' in typename or 'FLOAT' in typename or 'NUMERIC' in typename:
|
||||
return '0'
|
||||
return '<%s>' % column.name
|
||||
|
||||
|
||||
def dependency_order(names):
|
||||
"""The given tables, ordered so a table never precedes what it references."""
|
||||
known = [t for t in IMPORTABLE if t in set(names)]
|
||||
return known
|
||||
|
||||
|
||||
def table_from_filename(filename):
|
||||
stem = re.sub(r'\.csv$', '', filename.strip(), flags=re.I)
|
||||
stem = stem.rsplit('/', 1)[-1].rsplit('\\', 1)[-1].lower()
|
||||
return stem
|
||||
Reference in New Issue
Block a user