"""Every third-party module the app imports at runtime must be a DECLARED dependency. The bug this exists to prevent: `shopdb/plugins/loader.py` imports `packaging`, which was never listed in requirements.in. Locally and in CI that import resolved anyway, because pytest depends on packaging -- so all 1159 tests passed. The Windows installer builds its venv from requirements.txt alone, with no test dependencies, so `import shopdb` raised ModuleNotFoundError and the whole application failed to start on a customer server. Test dependencies are exactly the blind spot: they are present everywhere the test suite runs and absent everywhere it does not. The check is static (AST) rather than a real import, so it covers plugins that are not installed at this site and code paths no test exercises. """ import ast import sys from pathlib import Path import pytest try: from importlib.metadata import packages_distributions except ImportError: # pragma: no cover - Python < 3.10 packages_distributions = None REPO = Path(__file__).resolve().parent.parent SCANNED = ('shopdb', 'plugins', 'scripts') # First-party top-level names. Imports of these resolve from the source tree. FIRST_PARTY = {'shopdb', 'plugins', 'scripts', 'migrations', 'tests', 'wsgi', 'conftest'} # Modules deliberately imported without being declared. Each needs a reason. ALLOWED_UNDECLARED = { # Provided by the interpreter's own bundled installer tooling, and only # touched by developer scripts that never run on a deployed site. 'pip', 'setuptools', 'pkg_resources', } def _normalise(name): """PEP 503 normalisation, so Flask_Migrate and flask-migrate compare equal.""" return name.lower().replace('_', '-').replace('.', '-') def _declared_distributions(): """Distribution names pinned in requirements.txt.""" text = (REPO / 'requirements.txt').read_text(encoding='utf-8') declared = set() for line in text.splitlines(): line = line.strip() if not line or line.startswith('#') or line.startswith('--'): continue # "name==1.2.3 ; marker \" -> "name" head = line.split('==', 1)[0].split(';', 1)[0].strip() if head: declared.add(_normalise(head)) return declared def _iter_source_files(): for top in SCANNED: root = REPO / top if not root.is_dir(): continue for path in root.rglob('*.py'): parts = set(path.parts) if {'node_modules', 'venv', '.venv', 'build'} & parts: continue yield path def _imported_top_level_modules(path): """Top-level module names imported unconditionally by `path`. Imports nested inside a `try:` are skipped: that is how this codebase spells an optional dependency, and a missing one is handled rather than fatal. Imports inside functions are still counted -- a deferred import of an undeclared package fails just as hard, only later. """ try: tree = ast.parse(path.read_text(encoding='utf-8'), filename=str(path)) except SyntaxError: pytest.fail('could not parse {}'.format(path.relative_to(REPO))) guarded = set() for node in ast.walk(tree): if isinstance(node, ast.Try): for child in ast.walk(node): guarded.add(id(child)) found = set() for node in ast.walk(tree): if id(node) in guarded: continue if isinstance(node, ast.Import): for alias in node.names: found.add(alias.name.split('.')[0]) elif isinstance(node, ast.ImportFrom): # level > 0 is a relative (first-party) import. if node.level == 0 and node.module: found.add(node.module.split('.')[0]) return found @pytest.mark.skipif(packages_distributions is None, reason='needs Python 3.10+') def test_runtime_imports_are_declared_dependencies(): declared = _declared_distributions() assert 'flask' in declared, 'requirements.txt did not parse as expected' module_to_dists = packages_distributions() stdlib = getattr(sys, 'stdlib_module_names', frozenset()) offenders = {} for path in _iter_source_files(): for module in _imported_top_level_modules(path): if module in FIRST_PARTY or module in stdlib or module in ALLOWED_UNDECLARED: continue if module.startswith('_'): continue dists = module_to_dists.get(module) if not dists: # Not installed in this environment, so its distribution cannot # be identified. Absence here is not evidence of a problem. continue if not any(_normalise(d) in declared for d in dists): rel = str(path.relative_to(REPO)) offenders.setdefault( '{} (from {})'.format(module, '/'.join(sorted(dists))), [] ).append(rel) if offenders: lines = [] for module, files in sorted(offenders.items()): shown = sorted(files)[:3] more = '' if len(files) <= 3 else ' (+{} more)'.format(len(files) - 3) lines.append(' {}\n {}{}'.format(module, ', '.join(shown), more)) pytest.fail( 'Imported at runtime but not declared in requirements.txt.\n' 'These resolve here only because a test dependency supplies them; a\n' 'production venv has no test dependencies and will fail to import.\n' 'Add them to requirements.in and recompile.\n\n' + '\n'.join(lines) ) def test_packaging_is_declared(): """Regression: the specific import that broke the Windows install. Kept separate from the sweep above because that one can only flag a module it can attribute to an installed distribution -- an environment without packaging would skip it silently, which is precisely the shipping case. """ assert 'packaging' in _declared_distributions(), ( 'shopdb/plugins/loader.py imports packaging at module scope. Without it ' 'declared, `import shopdb` fails on any venv built from requirements.txt.' )