"""Bulk-load a site's starting data from CSV files.

WHY THIS EXISTS. Adopting a site means getting its existing asset register into
ShopDB. The HTTP import surface (docs/IMPORT-API.md) is the right tool when
there is a source system to read from and someone able to script against it. A
sister site with a spreadsheet and no developer needs something else, and that
is the common case.

THE DESIGN PROBLEM, which a hand-written template set gets wrong. A CSV row for
an asset has to say where the asset is. The database stores `locationid`, an
integer. Nobody types integers into a spreadsheet correctly, and requiring it
means the operator must first import locations, read back the generated ids, and
paste them into the asset sheet. That is not a workflow anyone completes.

So every foreign key here accepts EITHER a numeric id OR the referenced row's
name, and names are resolved at import time:

    assetnumber,assettypeid,locationid,businessunitid
    CMM-01,Measuring Tool,Building 1 Bay 3,Quality

The column keeps its database name - the convention in CONTRIBUTING.md - while
the VALUE is whatever the operator actually knows. A name that does not resolve
is a row error naming the column, the value, and where to fix it, not a foreign
key constraint violation from three layers down.

IDEMPOTENCE. Each table declares a natural key (`assetnumber`, `locationname`,
...). Re-importing a file updates the matched rows rather than duplicating them,
so a site can correct its spreadsheet and run it again - which they will.

SAFETY. Validation is a separate pass over the whole file BEFORE anything is
written, and the default is a dry run. An import either applies completely or
not at all.
"""
from __future__ import annotations

import csv
import io
import re
from collections import OrderedDict

from sqlalchemy import inspect as sqla_inspect

from shopdb.extensions import db


class ImportError_(Exception):
    """A problem with the request itself, not with a row."""


# The tables a site may load, in dependency order: a table never appears before
# something it references. `label` is the human-facing column - the one an
# operator would write in a spreadsheet, and the one used to resolve this table
# when another table points at it. `natural` is what makes a re-import an update
# rather than a duplicate.
#
# Deliberately NOT every table. Transactional and derived data (audit rows,
# reports, enforcement history) has no business arriving by spreadsheet, and
# offering it would invite someone to try.
IMPORTABLE = OrderedDict([
    ('assetstatuses',    {'label': 'status',        'natural': 'status'}),
    ('assettypes',       {'label': 'assettype',     'natural': 'assettype'}),
    ('locationtypes',    {'label': 'locationtype',  'natural': 'locationtype'}),
    ('modeltypes',       {'label': 'modeltype',     'natural': 'modeltype'}),
    ('computertypes',    {'label': 'computertype',  'natural': 'computertype'}),
    ('machinetypes',     {'label': 'machinetype',   'natural': 'machinetype'}),
    ('businessunits',    {'label': 'businessunit',  'natural': 'businessunit'}),
    ('locations',        {'label': 'locationname',  'natural': 'locationname'}),
    ('vendors',          {'label': 'vendor',        'natural': 'vendor'}),
    ('models',           {'label': 'modelnumber',   'natural': 'modelnumber'}),
    ('operatingsystems', {'label': 'osname',        'natural': 'osname'}),
    ('assets',           {'label': 'assetnumber',   'natural': 'assetnumber'}),
    ('computers',        {'label': 'hostname',      'natural': 'assetid'}),
    ('machines',         {'label': None,            'natural': 'assetid'}),
])

# Columns a CSV may never set: surrogate keys the database owns, and audit
# stamps. Accepting an id from a spreadsheet is how two sites end up with
# colliding primary keys the first time anyone merges data.
NEVER_IMPORT = {'createddate', 'modifieddate'}

# Credentials are never taken from a file. A CSV that carries password hashes
# gets mailed around, and one that carries plaintext is worse. Users are created
# with a random password and must change it at first login.
PASSWORD_COLUMNS = {'passwordhash', 'password'}

TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'}
FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''}


def table_for(name):
    table = db.metadata.tables.get(name)
    if table is None:
        raise ImportError_(
            "no such table: %s\nImportable: %s" % (name, ', '.join(IMPORTABLE)))
    return table


def assert_schema_ready(tablename):
    """Fail with a sentence, not a traceback, when the schema is not there yet.

    Running this against a database that has never been migrated is an easy
    mistake - a fresh site, or the wrong DATABASE_URL - and SQLAlchemy's answer
    is a wall of stack frames ending in 'no such table'. The operator needs to
    be told to run the migrations.
    """
    # The SESSION's connection, not db.engine. Taking a separate connection ends
    # up rolling back the session's in-flight transaction under a shared-
    # connection pool, which silently discarded rows that had just been written.
    inspector = sqla_inspect(db.session.connection())
    if not inspector.has_table(tablename):
        raise ImportError_(
            "the '%s' table does not exist in this database.\n"
            "  Run 'flask db upgrade' first, and check DATABASE_URL points at the\n"
            "  site you meant." % tablename)


def importable_columns(tablename):
    """Columns an operator may set, in a stable order.

    Primary keys are excluded - they are the database's - EXCEPT where the key
    is also a foreign key, as on the subtype tables where `assetid` both
    identifies the row and points at its asset.
    """
    table = table_for(tablename)
    primary = {c.name for c in table.primary_key}
    out = []
    for column in table.columns:
        if column.name in NEVER_IMPORT:
            continue
        if column.name in primary and not column.foreign_keys:
            continue
        if column.name in PASSWORD_COLUMNS:
            continue
        out.append(column)
    return out


def required_columns(tablename):
    primary = {c.name for c in table_for(tablename).primary_key}
    return [
        c.name for c in importable_columns(tablename)
        if not c.nullable and c.name not in primary
        and c.default is None and c.server_default is None
    ]


def _fk_target(column):
    """(tablename, idcolumn) this column points at, or None."""
    if not column.foreign_keys:
        return None
    target = list(column.foreign_keys)[0].column
    return target.table.name, target.name


def _label_column(tablename):
    """The human-readable column for a table, used to resolve references to it.

    Derived where possible so this does not need maintaining: the schema names
    reference tables' label column after the table itself (assettypes.assettype,
    locationtypes.locationtype). IMPORTABLE overrides where that rule does not
    hold.
    """
    spec = IMPORTABLE.get(tablename)
    if spec and spec.get('label'):
        return spec['label']
    table = db.metadata.tables.get(tablename)
    if table is None:
        return None
    singular = tablename[:-1] if tablename.endswith('s') else tablename
    for candidate in (singular, singular + 'name', tablename + 'name'):
        if candidate in table.columns:
            return candidate
    return None


def _coerce(column, raw):
    """Turn a CSV string into something the column will accept."""
    text = (raw or '').strip()
    typename = column.type.__class__.__name__.upper()

    if text == '':
        return None

    if 'BOOL' in typename:
        low = text.lower()
        if low in TRUE_VALUES:
            return True
        if low in FALSE_VALUES:
            return False
        raise ValueError("expected 1 or 0, got '%s'" % text)

    if 'INT' in typename:
        try:
            return int(text)
        except ValueError:
            raise ValueError("expected a whole number, got '%s'" % text)

    if 'FLOAT' in typename or 'NUMERIC' in typename or 'DECIMAL' in typename:
        try:
            return float(text)
        except ValueError:
            raise ValueError("expected a number, got '%s'" % text)

    if 'DATE' in typename or 'TIME' in typename:
        from datetime import datetime
        for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'):
            try:
                return datetime.strptime(text, fmt)
            except ValueError:
                continue
        raise ValueError("expected a date like 2026-08-04, got '%s'" % text)

    length = getattr(column.type, 'length', None)
    if length and len(text) > length:
        raise ValueError("longer than the column allows (%d > %d characters)" % (len(text), length))
    return text


class Resolver:
    """Turns names into ids, remembering what it has already looked up.

    Rows created earlier in the SAME import are resolvable, so one file can
    reference another that was loaded moments ago without a commit in between.
    """

    def __init__(self):
        self._cache = {}

    def resolve(self, tablename, idcolumn, value):
        if value is None:
            return None
        # A number is taken at face value: exports round-trip, and some sites do
        # know their ids.
        if isinstance(value, int) or (isinstance(value, str) and value.strip().isdigit()):
            return int(value)

        label = _label_column(tablename)
        if not label:
            raise ValueError(
                "cannot look up %s by name - give the numeric %s instead" % (tablename, idcolumn))

        key = (tablename, str(value).strip().lower())
        if key in self._cache:
            return self._cache[key]

        table = db.metadata.tables[tablename]
        row = db.session.execute(
            table.select().where(
                db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower())
        ).first()
        if row is None:
            raise ValueError(
                "nothing in %s is named '%s' - add it to %s.csv, or import that file first"
                % (tablename, value, tablename))
        found = getattr(row, idcolumn)
        self._cache[key] = found
        return found

    def remember(self, tablename, name, rowid):
        if name is None:
            return
        self._cache[(tablename, str(name).strip().lower())] = rowid


class RowProblem(object):
    def __init__(self, line, column, message):
        self.line = line
        self.column = column
        self.message = message

    def __str__(self):
        where = "line %d" % self.line
        if self.column:
            where += ", column '%s'" % self.column
        return "%s: %s" % (where, self.message)


class ImportResult(object):
    def __init__(self, tablename):
        self.tablename = tablename
        self.created = 0
        self.updated = 0
        self.problems = []
        self.skipped = 0

    @property
    def ok(self):
        return not self.problems

    def summary(self):
        return "%s: %d new, %d updated, %d problem(s)" % (
            self.tablename, self.created, self.updated, len(self.problems))


def read_rows(text):
    """Rows as dicts, with the sample/comment lines the templates carry removed."""
    reader = csv.reader(io.StringIO(text))
    header = None
    rows = []
    for number, fields in enumerate(reader, 1):
        if not fields or all(not f.strip() for f in fields):
            continue
        if fields[0].lstrip().startswith('#'):
            continue
        if header is None:
            # Strip a UTF-8 BOM if one survived: Excel writes it, and a header
            # named '\ufeffassetnumber' would match nothing.
            header = [f.strip().lstrip('\ufeff') for f in fields]
            continue
        # A short row is padded rather than rejected: trailing empty cells are
        # routinely dropped by spreadsheet exports.
        padded = list(fields) + [''] * (len(header) - len(fields))
        rows.append((number, dict(zip(header, padded))))
    if header is None:
        raise ImportError_('the file has no header row')
    return header, rows


def import_csv(tablename, text, resolver=None, commit=False):
    """Validate a CSV against one table and write it into the current transaction.

    Validation runs over EVERY row first. Nothing is written unless the whole
    file is clean, so a typo on line 400 does not leave 399 rows imported.

    Writes always go into the SESSION; this never commits. The caller commits or
    rolls back, which is what makes a dry run over a whole folder work: assets
    can resolve the locations the previous file just created, because they are
    really there inside the transaction, and the rollback removes them again.
    Skipping the writes on a dry run instead made every cross-file reference
    fail, which is the one thing a folder-wide check exists to verify.

    `commit` is retained only for callers that want the old explicit read.
    """
    table = table_for(tablename)
    spec = IMPORTABLE.get(tablename)
    if spec is None:
        raise ImportError_(
            "%s is not importable.\nImportable: %s" % (tablename, ', '.join(IMPORTABLE)))

    assert_schema_ready(tablename)
    resolver = resolver or Resolver()
    result = ImportResult(tablename)
    header, rows = read_rows(text)

    allowed = {c.name: c for c in importable_columns(tablename)}
    unknown = [h for h in header if h not in allowed]
    if unknown:
        known = ', '.join(sorted(allowed))
        result.problems.append(RowProblem(
            1, None, "unknown column(s): %s\n  this table accepts: %s" % (', '.join(unknown), known)))
        return result

    missing = [c for c in required_columns(tablename) if c not in header]
    if missing:
        result.problems.append(RowProblem(
            1, None, "required column(s) missing: %s" % ', '.join(missing)))
        return result

    natural = spec['natural']
    if natural not in header:
        result.problems.append(RowProblem(
            1, None, "the '%s' column identifies each row and must be present" % natural))
        return result

    prepared = []
    for line, raw in rows:
        values = {}
        failed = False
        for name, column in allowed.items():
            if name not in raw:
                continue
            target = _fk_target(column)
            # RESOLVE BEFORE COERCING. A foreign key column is an integer, so
            # coercing first rejects every name with "expected a whole number" -
            # which defeats the entire point of accepting names. Resolution
            # handles the numeric case itself.
            if target:
                text = (raw[name] or '').strip()
                if text == '':
                    values[name] = None
                    continue
                try:
                    values[name] = resolver.resolve(target[0], target[1], text)
                except ValueError as exc:
                    result.problems.append(RowProblem(line, name, str(exc)))
                    failed = True
                continue
            try:
                values[name] = _coerce(column, raw[name])
            except ValueError as exc:
                result.problems.append(RowProblem(line, name, str(exc)))
                failed = True

        if failed:
            continue
        if values.get(natural) in (None, ''):
            result.problems.append(RowProblem(line, natural, 'must have a value'))
            continue
        prepared.append((line, values))

    if not result.ok:
        return result

    # Second pass: decide insert vs update. Counted even on a dry run, because
    # "this will change 4000 rows" is the number an operator needs before saying
    # yes.
    label = _label_column(tablename)
    for line, values in prepared:
        existing = db.session.execute(
            table.select().where(table.c[natural] == values[natural])).first()
        if existing is None:
            result.created += 1
            inserted = db.session.execute(table.insert().values(**values))
            newid = inserted.inserted_primary_key[0] if inserted.inserted_primary_key else None
            if label and label in values:
                resolver.remember(tablename, values[label], newid)
        else:
            result.updated += 1
            db.session.execute(
                table.update().where(table.c[natural] == values[natural]).values(**values))
            if label and label in values:
                primary = list(table.primary_key)[0].name
                resolver.remember(tablename, values[label], getattr(existing, primary))

    return result


def generate_template(tablename):
    """A header row plus a commented example, built from the live schema.

    Generated rather than maintained: a hand-written template set drifts on the
    next migration, and silently - the file still looks right.
    """
    columns = importable_columns(tablename)
    required = set(required_columns(tablename))
    header = [c.name for c in columns]

    notes = []
    for column in columns:
        bits = []
        bits.append('REQUIRED' if column.name in required else 'optional')
        target = _fk_target(column)
        if target:
            label = _label_column(target[0])
            if label:
                bits.append("name from %s.csv (or a numeric id)" % target[0])
            else:
                bits.append("numeric %s id" % target[0])
        else:
            typename = column.type.__class__.__name__.upper()
            if 'BOOL' in typename:
                bits.append('1 or 0')
            elif 'DATE' in typename or 'TIME' in typename:
                bits.append('YYYY-MM-DD')
            elif getattr(column.type, 'length', None):
                bits.append('text, max %d' % column.type.length)
        notes.append("#   %-24s %s" % (column.name, ', '.join(bits)))

    spec = IMPORTABLE[tablename]
    lines = [
        "# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename,
        "#",
        "# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'],
        "# than creating a second one, so it is safe to correct this file and run again.",
        "#",
        "# Columns:",
    ] + notes + [
        "#",
        "# Lines starting with # are ignored. Delete the example row before importing.",
        ','.join(header),
        '# ' + ','.join(_example(c, required) for c in columns),
    ]
    return '\n'.join(lines) + '\n'


def _example(column, required):
    target = _fk_target(column)
    if target:
        label = _label_column(target[0])
        return ('<%s name>' % target[0][:-1]) if label else '<%s id>' % target[0]
    typename = column.type.__class__.__name__.upper()
    if 'BOOL' in typename:
        return '1'
    if 'DATE' in typename or 'TIME' in typename:
        return '2026-08-04'
    if 'INT' in typename or 'FLOAT' in typename or 'NUMERIC' in typename:
        return '0'
    return '<%s>' % column.name


def dependency_order(names):
    """The given tables, ordered so a table never precedes what it references."""
    known = [t for t in IMPORTABLE if t in set(names)]
    return known


def table_from_filename(filename):
    stem = re.sub(r'\.csv$', '', filename.strip(), flags=re.I)
    stem = stem.rsplit('/', 1)[-1].rsplit('\\', 1)[-1].lower()
    return stem
