shopdb-flask/tests/test_runtime_dependencies.py

"""Every third-party module the app imports at runtime must be a DECLARED dependency.

The bug this exists to prevent: `shopdb/plugins/loader.py` imports `packaging`,
which was never listed in requirements.in. Locally and in CI that import
resolved anyway, because pytest depends on packaging -- so all 1159 tests
passed. The Windows installer builds its venv from requirements.txt alone, with
no test dependencies, so `import shopdb` raised ModuleNotFoundError and the
whole application failed to start on a customer server.

Test dependencies are exactly the blind spot: they are present everywhere the
test suite runs and absent everywhere it does not.

The check is static (AST) rather than a real import, so it covers plugins that
are not installed at this site and code paths no test exercises.
"""

import ast
import sys
from pathlib import Path

import pytest

try:
    from importlib.metadata import packages_distributions
except ImportError:  # pragma: no cover - Python < 3.10
    packages_distributions = None

REPO = Path(__file__).resolve().parent.parent
SCANNED = ('shopdb', 'plugins', 'scripts')

# First-party top-level names. Imports of these resolve from the source tree.
FIRST_PARTY = {'shopdb', 'plugins', 'scripts', 'migrations', 'tests', 'wsgi', 'conftest'}

# Modules deliberately imported without being declared. Each needs a reason.
ALLOWED_UNDECLARED = {
    # Provided by the interpreter's own bundled installer tooling, and only
    # touched by developer scripts that never run on a deployed site.
    'pip',
    'setuptools',
    'pkg_resources',
}


def _normalise(name):
    """PEP 503 normalisation, so Flask_Migrate and flask-migrate compare equal."""
    return name.lower().replace('_', '-').replace('.', '-')


def _declared_distributions():
    """Distribution names pinned in requirements.txt."""
    text = (REPO / 'requirements.txt').read_text(encoding='utf-8')
    declared = set()
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith('#') or line.startswith('--'):
            continue
        # "name==1.2.3 ; marker \" -> "name"
        head = line.split('==', 1)[0].split(';', 1)[0].strip()
        if head:
            declared.add(_normalise(head))
    return declared


def _iter_source_files():
    for top in SCANNED:
        root = REPO / top
        if not root.is_dir():
            continue
        for path in root.rglob('*.py'):
            parts = set(path.parts)
            if {'node_modules', 'venv', '.venv', 'build'} & parts:
                continue
            yield path


def _imported_top_level_modules(path):
    """Top-level module names imported unconditionally by `path`.

    Imports nested inside a `try:` are skipped: that is how this codebase spells
    an optional dependency, and a missing one is handled rather than fatal.
    Imports inside functions are still counted -- a deferred import of an
    undeclared package fails just as hard, only later.
    """
    try:
        tree = ast.parse(path.read_text(encoding='utf-8'), filename=str(path))
    except SyntaxError:
        pytest.fail('could not parse {}'.format(path.relative_to(REPO)))

    guarded = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Try):
            for child in ast.walk(node):
                guarded.add(id(child))

    found = set()
    for node in ast.walk(tree):
        if id(node) in guarded:
            continue
        if isinstance(node, ast.Import):
            for alias in node.names:
                found.add(alias.name.split('.')[0])
        elif isinstance(node, ast.ImportFrom):
            # level > 0 is a relative (first-party) import.
            if node.level == 0 and node.module:
                found.add(node.module.split('.')[0])
    return found


@pytest.mark.skipif(packages_distributions is None, reason='needs Python 3.10+')
def test_runtime_imports_are_declared_dependencies():
    declared = _declared_distributions()
    assert 'flask' in declared, 'requirements.txt did not parse as expected'

    module_to_dists = packages_distributions()
    stdlib = getattr(sys, 'stdlib_module_names', frozenset())

    offenders = {}
    for path in _iter_source_files():
        for module in _imported_top_level_modules(path):
            if module in FIRST_PARTY or module in stdlib or module in ALLOWED_UNDECLARED:
                continue
            if module.startswith('_'):
                continue
            dists = module_to_dists.get(module)
            if not dists:
                # Not installed in this environment, so its distribution cannot
                # be identified. Absence here is not evidence of a problem.
                continue
            if not any(_normalise(d) in declared for d in dists):
                rel = str(path.relative_to(REPO))
                offenders.setdefault(
                    '{} (from {})'.format(module, '/'.join(sorted(dists))), []
                ).append(rel)

    if offenders:
        lines = []
        for module, files in sorted(offenders.items()):
            shown = sorted(files)[:3]
            more = '' if len(files) <= 3 else ' (+{} more)'.format(len(files) - 3)
            lines.append('  {}\n      {}{}'.format(module, ', '.join(shown), more))
        pytest.fail(
            'Imported at runtime but not declared in requirements.txt.\n'
            'These resolve here only because a test dependency supplies them; a\n'
            'production venv has no test dependencies and will fail to import.\n'
            'Add them to requirements.in and recompile.\n\n' + '\n'.join(lines)
        )


def test_packaging_is_declared():
    """Regression: the specific import that broke the Windows install.

    Kept separate from the sweep above because that one can only flag a module
    it can attribute to an installed distribution -- an environment without
    packaging would skip it silently, which is precisely the shipping case.
    """
    assert 'packaging' in _declared_distributions(), (
        'shopdb/plugins/loader.py imports packaging at module scope. Without it '
        'declared, `import shopdb` fails on any venv built from requirements.txt.'
    )