Release 0.8.0: Windows installer, twelve fixes from a real Server 2019 install, operator docs

shopdb/cli/__init__.py

@@ -825,3 +825,160 @@ def seed_demo_clear(yes):
     click.echo(click.style(
         f"Removed {len(demo_ids)} demo assets and "
         f"{parts_deleted} sample parts.", fg='green'))
+
+
+@click.group('csv')
+def csv_cli():
+    """Load a site's starting data from CSV files."""
+    pass
+
+
+@csv_cli.command('templates')
+@click.option('--out', 'outdir', default='csv-templates',
+              help='Directory to write the templates into.')
+@with_appcontext
+def csv_templates(outdir):
+    """Write a CSV template per importable table, generated from the schema.
+
+    Generated rather than kept by hand: a maintained template set drifts on the
+    next migration and does so silently, since the file still looks correct.
+    """
+    import os
+    from shopdb.core.services.csvimport import IMPORTABLE, generate_template
+
+    if not os.path.isdir(outdir):
+        os.makedirs(outdir)
+    for tablename in IMPORTABLE:
+        path = os.path.join(outdir, tablename + '.csv')
+        with open(path, 'w', encoding='utf-8') as handle:
+            handle.write(generate_template(tablename))
+        click.echo('  %s' % path)
+
+    readme = os.path.join(outdir, 'README.txt')
+    with open(readme, 'w', encoding='utf-8') as handle:
+        handle.write(
+            'ShopDB-Flask import templates\n'
+            '=============================\n\n'
+            'Generated from the live database schema. Every column here exists;\n'
+            'every required column is marked.\n\n'
+            'Fill in the ones you need - you do not need all of them.\n\n'
+            'Foreign keys take a NAME or a numeric id. Write the name:\n'
+            '    locationid  ->  Building 1 Bay 3\n'
+            '    vendorid    ->  Haas Automation\n'
+            'The importer resolves it, and tells you which row and column to fix\n'
+            'if the name is not found.\n\n'
+            'Import the whole folder at once and order is handled for you:\n\n'
+            '    flask csv import --dir .            (checks only, changes nothing)\n'
+            '    flask csv import --dir . --commit   (applies)\n\n'
+            'Nothing is written unless every row passes, so a mistake on line 400\n'
+            'does not leave 399 rows half-imported.\n\n'
+            'User accounts are deliberately not importable here: passwords do not\n'
+            'belong in a spreadsheet.\n')
+    click.echo('  %s' % readme)
+    click.echo('')
+    click.echo(click.style('%d templates written to %s' % (len(IMPORTABLE), outdir),
+                           fg='green', bold=True))
+
+
+@csv_cli.command('import')
+@click.option('--file', 'path', default=None, help='One CSV file.')
+@click.option('--dir', 'directory', default=None,
+              help='A folder of CSVs, imported in dependency order.')
+@click.option('--table', 'tablename', default=None,
+              help='Target table. Defaults to the file name.')
+@click.option('--commit', is_flag=True, default=False,
+              help='Apply the changes. Without this, nothing is written.')
+@with_appcontext
+def csv_import(path, directory, tablename, commit):
+    """Validate CSVs and, with --commit, load them.
+
+    Dry run by default. The report is the same either way, so what you review is
+    what you get.
+    """
+    import os
+    from shopdb.extensions import db
+    from sqlalchemy.exc import SQLAlchemyError
+    from shopdb.core.services.csvimport import (
+        ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
+
+    if not path and not directory:
+        raise click.UsageError('give --file or --dir')
+
+    jobs = []
+    if path:
+        jobs.append((tablename or table_from_filename(os.path.basename(path)), path))
+    else:
+        found = {}
+        for name in os.listdir(directory):
+            if not name.lower().endswith('.csv'):
+                continue
+            found[table_from_filename(name)] = os.path.join(directory, name)
+        ordered = dependency_order(list(found))
+        unknown = sorted(set(found) - set(ordered))
+        for name in ordered:
+            jobs.append((name, found[name]))
+        if unknown:
+            click.echo(click.style(
+                'skipping (not importable): %s' % ', '.join(unknown), fg='yellow'))
+
+    if not jobs:
+        click.echo('nothing to do - no CSV files found')
+        return
+
+    click.echo(click.style(
+        'Checking %d file(s)%s' % (len(jobs), '' if commit else ' - DRY RUN, nothing will be written'),
+        bold=True))
+    click.echo('')
+
+    resolver = Resolver()
+    results = []
+    failed = False
+    for name, filepath in jobs:
+        with open(filepath, 'r', encoding='utf-8-sig') as handle:
+            text = handle.read()
+        try:
+            # Every file is applied inside ONE transaction, so a failure part way
+            # through a folder rolls the whole run back rather than leaving the
+            # site half-populated.
+            result = import_csv(name, text, resolver=resolver, commit=commit)
+        except ImportError_ as exc:
+            click.echo(click.style('  %-18s %s' % (name, exc), fg='red'))
+            failed = True
+            continue
+        except SQLAlchemyError as exc:
+            # Anything the database itself refuses. The operator gets the cause
+            # in one line rather than a traceback they cannot act on.
+            db.session.rollback()
+            click.echo(click.style('  %-18s database error: %s'
+                                   % (name, str(exc).split(chr(10))[0]), fg='red'))
+            failed = True
+            continue
+        results.append(result)
+        colour = 'green' if result.ok else 'red'
+        click.echo(click.style('  ' + result.summary(), fg=colour))
+        for problem in result.problems[:20]:
+            click.echo(click.style('      %s' % problem, fg='red'))
+        if len(result.problems) > 20:
+            click.echo(click.style('      ... and %d more' % (len(result.problems) - 20), fg='red'))
+        if not result.ok:
+            failed = True
+
+    click.echo('')
+    if failed:
+        db.session.rollback()
+        click.echo(click.style('Nothing was imported. Fix the problems above and run again.',
+                               fg='red', bold=True))
+        raise SystemExit(1)
+
+    total_new = sum(r.created for r in results)
+    total_upd = sum(r.updated for r in results)
+    if commit:
+        db.session.commit()
+        click.echo(click.style('Imported: %d new, %d updated.' % (total_new, total_upd),
+                               fg='green', bold=True))
+    else:
+        db.session.rollback()
+        click.echo(click.style(
+            'Looks good: %d would be created, %d updated.' % (total_new, total_upd),
+            fg='green', bold=True))
+        click.echo('Run again with --commit to apply.')