shopdb-flask/tests/test_plugin_migrations.py

"""Per-plugin Alembic migration-chain guard tests.

Ownership cutover (ADR-008): the core Alembic chain created every table that
exists through its head (`7d16_directoryemployees`), including the plugin
tables. From that point on, each bundled plugin that owns tables carries its
own chain under `plugins/<name>/migrations/`. The `0001` migration in each
chain is a stamp-only no-op anchor: the core chain already built the tables, so
there is nothing to create; the anchor just gives the plugin chain a base that
`flask plugin upgrade-all` stamps into `alembic_version_<plugin>`.

These tests pin that contract:
  * PLUGIN_TABLE_OWNERS stays in sync with what the models declare.
  * Every plugin that owns tables has a valid single-head chain.
  * The anchor migrations are genuine no-ops.
  * `flask plugin upgrade-all` runs clean on a fresh DB and is idempotent.
"""
import ast
from pathlib import Path

import pytest

from shopdb.plugins.alembic_template import (
    PLUGIN_TABLE_OWNERS,
    _get_plugin_metadata,
)

PLUGINS_DIR = Path(__file__).resolve().parent.parent / 'plugins'

# Plugins that own tables carry a migration chain; blueprint-only plugins do
# not. Today every bundled plugin owns tables, so this is the full set.
TABLE_OWNING_PLUGINS = tuple(sorted(PLUGIN_TABLE_OWNERS))

# The ADR-008 cutover froze this exact set of ten plugins whose tables the core
# chain had already created. Their 0001 revision is a stamp-only no-op anchor.
# Plugins built AFTER the cutover (e.g. measuringtools) are NOT in this list:
# their 0001 is a real baseline that genuinely creates their tables, so the
# no-op assertion must not apply to them. This is a frozen list on purpose - a
# newly discovered plugin does not silently get treated as a cutover no-op.
CUTOVER_PLUGINS = (
    'computers', 'employees', 'knowledgebase', 'machines', 'network',
    'notifications', 'printers', 'slides', 'usb', 'warranty',
)

# Expected head revision id per table-owning plugin, so the upgrade-all test can
# check both the cutover anchors and post-cutover baselines. Cutover plugins
# stamp '<plugin>0001anchor'; measuringtools stamps its real baseline id.
EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS}
EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline'
# computers adds the Entra/Azure device identity columns (azuredeviceid, then
# azureobjectid + azuredevicecategory), then the Configuration/Business policy
# tables, then the break-glass policyactionholds table, on top of its anchor, so
# its head is not the f-string default. 0006 widens policyactionholds.expiresat
# to nullable, where null means an indefinite hold that never self-clears.
EXPECTED_HEAD_REVISION['computers'] = 'computers0006holdindefinite'
# backups is also post-cutover: its 0001 really creates backuprevisions, and
# 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup
# otherwise throws away.
EXPECTED_HEAD_REVISION['backups'] = 'backups0003clearlastseen'
# guides is a NEW plugin, so its 0001 builds tables rather than stamping an
# anchor over core-owned ones.
EXPECTED_HEAD_REVISION['guides'] = 'guides0001baseline'
# geenforce adds the content-addressed blob store (manifestblobs) on top of its
# baseline.
EXPECTED_HEAD_REVISION['geenforce'] = 'geenforce0004minlib'
# printers adds the printersupplyalerts crossing-state table on top of its
# anchor, then the exact INF driver name Add-PrinterDriver needs, a vendor on
# the driver, and finally printerobservedqueues - what a bay reported it HAS,
# kept apart from what it was assigned.
EXPECTED_HEAD_REVISION['printers'] = 'printers0005observedqueues'
# machines (renamed from equipment) keeps its original anchor id and adds the
# rename revision on top, so its head is not the f-string default.
EXPECTED_HEAD_REVISION['machines'] = 'machines0002rename'
# employees adds the photofilename column on top of its cutover anchor.
EXPECTED_HEAD_REVISION['employees'] = 'employees0002photo'
# usb drops the dead usbcheckouts.machineid column on top of its anchor, then
# adds scannedviruses so an imported external history keeps the scan result -
# the field a removable-media audit actually asks about - and finally records
# who handed a drive back, which is not always who took it out.
EXPECTED_HEAD_REVISION['usb'] = 'usb0004checkinidentity'
# network links a device to a catalog model, which is where its photo comes
# from - machines, PCs and printers already had that link.
EXPECTED_HEAD_REVISION['network'] = 'network0003prefix'
# printedparts is post-cutover: its 0001 really creates its tables; 0004 adds
# the per-transaction revision column.
EXPECTED_HEAD_REVISION['printedparts'] = 'printedparts0004txnrev'
# Built after the ADR-008 cutover, so its 0001 genuinely creates the table
# rather than stamping an anchor. Tech Tools owns a table now because secure
# send needs one; every other tool in that plugin is browser-side.
EXPECTED_HEAD_REVISION['tools'] = 'tools0001securesend'
# notifications indexes businessunitid, then adds the per-type grace window and
# the shared board category, then the fleet-alert columns plus the per-asset
# acknowledgement table the endpoint agent writes to.
EXPECTED_HEAD_REVISION['notifications'] = 'notifications0006fleetalerts'
# warranty adds the proof-of-cover document columns on top of its anchor.
EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof'
# endpoints is a NEW post-cutover plugin: its 0001 genuinely creates the
# endpointstates + endpointevents tables the core chain never owned, 0002
# adds the endpointcommands queue plus the MTConnect state columns, 0003
# adds the generic command target/targetkind columns, 0004 adds the
# endpointapplications SBOM snapshot table, 0005 pins a command sha256, 0006
# seeds the reusable monitor catalog, 0007 ships the shopfloor monitor policy
# bundle, 0008 adds the warn-only reconciler drift cache column, and 0009 adds
# the operatorreports table for tray issue reports.
# 0010 renames the shipped business policy off the originating site's
# designator on installs that already stamped 0007 (ADR-015).
# 0011 adds the trayversion cache column to endpointstates.
# 0012 adds endpointevents.eventrecordid plus the unique index that dedups a
# replayed Windows event record.
# 0013 adds the inventory columns.
# 0014 adds endpointstates.diskhealthjson, and 0015 adds the system telemetry
# columns (boot info, reboot reasons, Defender health, NIC link state).
# 0016 adds the Dell update tables.
# 0017 adds the health rollup: endpointhealthstates, endpointhealthtransitions
# and endpointmonitorimpacts.
# 0018 adds endpointlocks: the admin quarantine refusing actions from non-admins
# on one endpoint, a caller-authority axis separate from the policy hold.
# 0019 splits the 0007 bundle into one shipped Configuration Policy per monitor
# and drops the shipped Business Policy, so sites compose their own.
# 0020 adds endpointapplications.appfingerprint for delta app updates. 0021 and
# 0022 are retired no-ops (client certificates, removed). 0023 drops their
# leftover asset columns, renames app_fingerprint, and adds per-PC credentials:
# endpointcredentials, endpointenrollrequests and endpointmigrationwaves.
EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0023credentials'

# Plugins built after the cutover: their 0001 baseline really creates tables the
# core chain never owned.
POST_CUTOVER_PLUGINS = tuple(p for p in PLUGIN_TABLE_OWNERS if p not in CUTOVER_PLUGINS)


def _declared_tablenames(plugin: str) -> set:
    """Scan a plugin's models package for every __tablename__ literal.

    Static parse (no import) so the test can compare what the code declares
    against PLUGIN_TABLE_OWNERS without side effects.
    """
    names = set()
    models_dir = PLUGINS_DIR / plugin / 'models'
    if not models_dir.exists():
        return names
    for source in models_dir.glob('*.py'):
        tree = ast.parse(source.read_text())
        for node in ast.walk(tree):
            if isinstance(node, ast.Assign):
                targets = [t.id for t in node.targets if isinstance(t, ast.Name)]
                if '__tablename__' in targets and isinstance(node.value, ast.Constant):
                    names.add(node.value.value)
    return names


def test_table_owners_match_declared_models():
    """Every plugin that declares a __tablename__ is registered in
    PLUGIN_TABLE_OWNERS, and vice versa. Catches a new plugin table that
    forgot to update the ownership map."""
    declared = {p.name for p in PLUGINS_DIR.iterdir()
                if p.is_dir() and _declared_tablenames(p.name)}
    assert declared == set(PLUGIN_TABLE_OWNERS), (
        f"PLUGIN_TABLE_OWNERS keys {set(PLUGIN_TABLE_OWNERS)} do not match "
        f"plugins declaring tables {declared}"
    )


@pytest.mark.parametrize('plugin', TABLE_OWNING_PLUGINS)
def test_bundled_plugin_has_table_owner_entry(plugin):
    """Every table-owning plugin appears in PLUGIN_TABLE_OWNERS with at least
    one table, documenting which tables it contributes to the schema."""
    assert plugin in PLUGIN_TABLE_OWNERS
    assert len(PLUGIN_TABLE_OWNERS[plugin]) > 0


@pytest.mark.parametrize('plugin', TABLE_OWNING_PLUGINS)
def test_owned_tables_match_declared_models(plugin):
    """The tables named in PLUGIN_TABLE_OWNERS are exactly the ones the
    plugin's models declare. Catches drift in either direction."""
    assert set(PLUGIN_TABLE_OWNERS[plugin]) == _declared_tablenames(plugin)


@pytest.mark.parametrize('plugin', TABLE_OWNING_PLUGINS)
def test_plugin_metadata_has_all_owned_tables(plugin, app):
    """The MetaData filtered to a plugin's owned tables actually contains
    every table named in PLUGIN_TABLE_OWNERS."""
    with app.app_context():
        md = _get_plugin_metadata(plugin)
        owned = set(PLUGIN_TABLE_OWNERS[plugin])
        missing = owned - set(md.tables.keys())
        assert not missing, f"Plugin {plugin}: owned tables not in metadata: {missing}"


@pytest.mark.parametrize('plugin', TABLE_OWNING_PLUGINS)
def test_plugin_has_migration_chain(plugin):
    """Every table-owning plugin has a migrations dir with env.py and exactly
    one anchor revision whose down_revision is None (a valid single-root
    chain)."""
    mig = PLUGINS_DIR / plugin / 'migrations'
    assert (mig / 'env.py').exists(), f"{plugin}: missing migrations/env.py"
    versions = sorted((mig / 'versions').glob('*.py'))
    assert versions, f"{plugin}: no version scripts"

    roots = []
    heads = set()
    down_revisions = set()
    revisions = set()
    for script in versions:
        tree = ast.parse(script.read_text())
        rev = down = None
        found_down = False
        for node in ast.walk(tree):
            if isinstance(node, ast.Assign):
                names = [t.id for t in node.targets if isinstance(t, ast.Name)]
                if 'revision' in names and isinstance(node.value, ast.Constant):
                    rev = node.value.value
                if 'down_revision' in names:
                    found_down = True
                    if isinstance(node.value, ast.Constant):
                        down = node.value.value
        assert rev, f"{plugin}: {script.name} has no revision id"
        # alembic_version_<plugin>.version_num is VARCHAR(32). A longer id
        # passes every SQLite test and then fails the real deploy on strict
        # MySQL with "Data too long for column 'version_num'", after the
        # migration body has already run. Caught
        # endpoints0019splitmonitorpolicies (33 chars) during a container roll.
        # The CORE chain's alembic_version is VARCHAR(128), so core ids are not
        # bound by this and are not checked here.
        assert len(rev) <= 32, (
            f"{plugin}: {script.name} revision id {rev!r} is {len(rev)} chars; "
            'alembic_version.version_num holds 32'
        )
        assert found_down, f"{plugin}: {script.name} has no down_revision"
        revisions.add(rev)
        if down is None:
            roots.append(rev)
        else:
            down_revisions.add(down)
    heads = revisions - down_revisions
    assert len(roots) == 1, f"{plugin}: expected 1 root revision, got {roots}"
    assert len(heads) == 1, f"{plugin}: chain must have a single head, got {heads}"


@pytest.mark.parametrize('plugin', CUTOVER_PLUGINS)
def test_anchor_migration_is_noop(plugin):
    """The 0001 anchor's upgrade() and downgrade() are pure no-ops: no DDL
    operations, just `pass`. The core chain owns the tables at cutover.

    Scoped to CUTOVER_PLUGINS only. A plugin built after the cutover ships a
    real baseline (measuringtools), which is deliberately NOT a no-op."""
    anchor = PLUGINS_DIR / plugin / 'migrations' / 'versions' / f'0001_{plugin}_anchor.py'
    assert anchor.exists(), f"{plugin}: missing 0001 anchor migration"
    tree = ast.parse(anchor.read_text())
    funcs = {n.name: n for n in ast.walk(tree)
             if isinstance(n, ast.FunctionDef) and n.name in ('upgrade', 'downgrade')}
    assert set(funcs) == {'upgrade', 'downgrade'}, f"{plugin}: anchor missing up/downgrade"
    for name, fn in funcs.items():
        # Body may only be a docstring/comment plus a bare `pass`. No calls.
        calls = [n for n in ast.walk(fn) if isinstance(n, ast.Call)]
        assert not calls, f"{plugin}: anchor {name}() is not a no-op (found calls)"


def test_upgrade_all_on_fresh_db_is_clean_and_idempotent(tmp_path, monkeypatch):
    """`flask plugin upgrade-all` on a fresh SQLite DB (after the core schema
    is created) stamps every plugin anchor without error, and a second run is
    a no-op. Mirrors the deploy sequence: `flask db upgrade` then
    `flask plugin upgrade-all`."""
    from sqlalchemy import inspect, text
    from shopdb.config import TestingConfig
    from shopdb import create_app
    from shopdb.extensions import db
    from shopdb.plugins import plugin_manager

    db_file = tmp_path / 'fresh.db'
    url = f'sqlite:///{db_file}'

    # Point the whole app (db engine + migration manager) at one file DB so the
    # anchor stamps land where the app can read them back.
    monkeypatch.setattr(TestingConfig, 'SQLALCHEMY_DATABASE_URI', url)

    # create_app repoints the process-wide plugin_manager singleton; snapshot
    # its wiring and restore it so later tests see the session app unchanged.
    saved = (plugin_manager._app, plugin_manager._db, plugin_manager.registry,
             plugin_manager.loader, plugin_manager.migration_manager,
             plugin_manager._registered_prefixes)
    try:
        app = create_app('testing')
        with app.app_context():
            db.create_all()  # stand in for the core `flask db upgrade`

            # db.create_all() over-creates: it builds EVERY table registered on
            # the metadata, including post-cutover plugin tables the core chain
            # would never own. Drop those so each post-cutover baseline creates
            # its own tables exactly as it does after a real core upgrade (where
            # the tables are simply absent). The cutover anchors are no-ops, so
            # their create_all-built tables stay put.
            insp0 = inspect(db.engine)
            for plugin in POST_CUTOVER_PLUGINS:
                # Drop by owned name (SQLite tolerates any order with no rows);
                # avoids resolving cross-metadata FKs via sorted_tables.
                for tablename in PLUGIN_TABLE_OWNERS[plugin]:
                    if insp0.has_table(tablename):
                        db.session.execute(text(f'DROP TABLE {tablename}'))
            db.session.commit()

            first = app.extensions['plugin_manager'].upgrade_all_plugins()
            second = app.extensions['plugin_manager'].upgrade_all_plugins()

            # A plugin that owns no tables (Tech Tools: every tool is
            # client-side) carries no chain, so upgrade-all reports it
            # 'no-migrations'. Assert that explicitly rather than letting it
            # widen the 'ok' check, which would also swallow a table-owning
            # plugin whose chain silently went missing.
            schemaless = set(first) - set(PLUGIN_TABLE_OWNERS)
            for name in schemaless:
                assert first[name] == 'no-migrations', (name, first[name])
                assert second[name] == 'no-migrations', (name, second[name])

            migrated = {name: status for name, status in first.items()
                        if name in PLUGIN_TABLE_OWNERS}
            assert set(migrated) == set(PLUGIN_TABLE_OWNERS)
            assert all(status == 'ok' for status in migrated.values()), migrated
            assert all(second[name] == 'ok' for name in PLUGIN_TABLE_OWNERS), second

            insp = inspect(db.engine)
            for plugin in PLUGIN_TABLE_OWNERS:
                version_table = f'alembic_version_{plugin}'
                assert insp.has_table(version_table), f"missing {version_table}"
                row = db.session.execute(
                    text(f'SELECT version_num FROM {version_table}')
                ).fetchone()
                assert row and row[0] == EXPECTED_HEAD_REVISION[plugin]
    finally:
        (plugin_manager._app, plugin_manager._db, plugin_manager.registry,
         plugin_manager.loader, plugin_manager.migration_manager,
         plugin_manager._registered_prefixes) = saved



# EVERY post-cutover chain, not just the single-baseline ones. This was once
# restricted to plugins whose whole chain is one baseline, on the reasoning that
# a longer chain meets a second problem - an ALTER whose column is already
# present - that guarding a create_table does not fix. True, but the conclusion
# was wrong: the answer is to guard the ALTERs too, not to stop testing the
# chains that have them. While this was narrowed, printedparts 0003 sat
# unguarded for a month and would fail upgrade-all on any create-all-built
# database, taking 0004 down with it, with no test able to see it.
#
# The invariant is now the whole one: a post-cutover chain must run to its head
# against a database whose tables were built from the models and stamped
# nowhere, and must do it twice.


@pytest.mark.parametrize('plugin', POST_CUTOVER_PLUGINS)
def test_baseline_skips_tables_that_already_exist(plugin, tmp_path, monkeypatch):
    """A baseline must stamp, not fail, when its table is already there.

    Reported: `flask plugin upgrade-all` failed the tools plugin with "table
    securesendsecrets already exists". Every post-cutover baseline was a plain
    op.create_table, correct only while a table's existence and its plugin's
    Alembic stamp can never disagree. They can - anything that builds tables
    from the models (db.create_all) stamps nothing, and a run that creates a
    table then fails before its stamp commits leaves the same split.

    The other upgrade-all test DROPS these tables before running, so it only
    ever covered the state where they are absent. This is the other state.
    """
    from sqlalchemy import inspect, text
    from shopdb.config import TestingConfig
    from shopdb import create_app
    from shopdb.extensions import db
    from shopdb.plugins import plugin_manager

    monkeypatch.setattr(TestingConfig, 'SQLALCHEMY_DATABASE_URI',
                        f'sqlite:///{tmp_path / "preexisting.db"}')

    saved = (plugin_manager._app, plugin_manager._db, plugin_manager.registry,
             plugin_manager.loader, plugin_manager.migration_manager,
             plugin_manager._registered_prefixes)
    try:
        app = create_app('testing')
        with app.app_context():
            db.create_all()

            # Leave THIS plugin's tables in place - that is the whole point -
            # and drop the other post-cutover ones so their own (longer) chains
            # run the way the sibling test already covers.
            insp0 = inspect(db.engine)
            for other in POST_CUTOVER_PLUGINS:
                if other == plugin:
                    continue
                for tablename in PLUGIN_TABLE_OWNERS[other]:
                    if insp0.has_table(tablename):
                        db.session.execute(text(f'DROP TABLE {tablename}'))
            db.session.commit()

            for tablename in PLUGIN_TABLE_OWNERS[plugin]:
                assert inspect(db.engine).has_table(tablename), (
                    f'{tablename} should have been built by create_all; '
                    f'this test is meaningless without it')

            first = app.extensions['plugin_manager'].upgrade_all_plugins()
            assert first[plugin] == 'ok', first[plugin]

            second = app.extensions['plugin_manager'].upgrade_all_plugins()
            assert second[plugin] == 'ok', second[plugin]

            insp = inspect(db.engine)
            row = db.session.execute(
                text(f'SELECT version_num FROM alembic_version_{plugin}')
            ).fetchone()
            assert row and row[0] == EXPECTED_HEAD_REVISION[plugin]

            # No second index over the same columns: create_all names indexes by
            # SQLAlchemy's convention and the baseline by this repo's, so a
            # name-keyed guard would have added a duplicate rather than skipped.
            for tablename in PLUGIN_TABLE_OWNERS[plugin]:
                seen = [tuple(ix.get('column_names') or ())
                        for ix in insp.get_indexes(tablename)]
                assert len(seen) == len(set(seen)), (
                    f'{tablename} has two indexes over the same columns: {seen}')
    finally:
        (plugin_manager._app, plugin_manager._db, plugin_manager.registry,
         plugin_manager.loader, plugin_manager.migration_manager,
         plugin_manager._registered_prefixes) = saved


def test_upgrade_all_seeds_permissions_and_links_them_to_scopes(
        tmp_path, monkeypatch):
    """`flask plugin upgrade-all` seeds plugin permissions AND relinks scopes.

    on_install/on_enable fire only on a state transition, so a permission
    declared by a LATER plugin version never reaches an already-installed site
    through them. Upgrade is the only remaining moment. Before this was wired,
    the name existed only after a manual `flask seed permissions`.

    The scope relink is the half that actually bites. Creating the Permission
    row only makes the name exist - the prebuilt scopes derive their grants from
    the catalog, so a permission seeded without a following seedscopes() belongs
    to nobody but admin. The plugin installs clean and its UI is invisible to
    every non-admin, which reads as a broken plugin rather than a seeding gap.
    """
    from shopdb.config import TestingConfig
    from shopdb import create_app
    from shopdb.extensions import db
    from shopdb.core.models import Permission, Role
    from shopdb.plugins import plugin_manager

    db_file = tmp_path / 'perms.db'
    monkeypatch.setattr(TestingConfig, 'SQLALCHEMY_DATABASE_URI',
                        f'sqlite:///{db_file}')

    saved = (plugin_manager._app, plugin_manager._db, plugin_manager.registry,
             plugin_manager.loader, plugin_manager.migration_manager,
             plugin_manager._registered_prefixes)
    try:
        app = create_app('testing')
        with app.app_context():
            db.create_all()
            Permission.seed()
            Role.seedscopes()
            db.session.commit()

            pm = app.extensions['plugin_manager']

            # Stand in for a permission that a later plugin version declares:
            # delete a plugin-owned view permission so upgrade-all has to put it
            # back the way it would for a genuinely new one.
            target = Permission.query.filter_by(name='endpoints.view').first()
            assert target is not None, 'fixture expects endpoints.view to exist'
            for role in list(target.roles):
                role.permissions.remove(target)
            db.session.delete(target)
            db.session.commit()
            assert Permission.query.filter_by(name='endpoints.view').first() is None

            pm.upgrade_all_plugins()

            # The row is back...
            restored = Permission.query.filter_by(name='endpoints.view').first()
            assert restored is not None, (
                'upgrade-all did not re-seed a missing plugin permission')

            # ...and it is actually granted to the read-capable scopes, not
            # merely present. This is the assertion that fails if seedscopes()
            # is dropped from the upgrade path.
            holders = {role.rolename for role in restored.roles}
            assert 'technician' in holders, (
                f'endpoints.view reseeded but not linked to any scope: {holders}')
    finally:
        (plugin_manager._app, plugin_manager._db, plugin_manager.registry,
         plugin_manager.loader, plugin_manager.migration_manager,
         plugin_manager._registered_prefixes) = saved