Declare packaging as a runtime dependency
shopdb/plugins/loader.py imports packaging.specifiers and packaging.version at module scope, but packaging was never listed in requirements.in. It was present in every development and CI environment as a transitive dependency of pytest, so the full suite passed while a venv built from requirements.txt alone could not import shopdb at all. The Windows installer builds exactly such a venv, so stage 3 failed on a customer server with ModuleNotFoundError: No module named 'packaging', after the runtime and all wheels had installed successfully. Add packaging to requirements.in, recompile the hashed lockfile, and add the wheel to the offline wheelhouse with the matching bundle-lock entry. The recompile also picked up newer uv formatting: inline environment markers on cffi and greenlet and shorter "via" comments. The pinned distribution set and every existing hash are unchanged. tests/test_runtime_dependencies.py guards the general case by scanning shopdb/, plugins/ and scripts/ for unconditional third-party imports and asserting each maps to a distribution pinned in requirements.txt. Test dependencies are the blind spot for this class of failure, since they are present wherever the suite runs and absent wherever it does not.
This commit is contained in:
159
tests/test_runtime_dependencies.py
Normal file
159
tests/test_runtime_dependencies.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""Every third-party module the app imports at runtime must be a DECLARED dependency.
|
||||
|
||||
The bug this exists to prevent: `shopdb/plugins/loader.py` imports `packaging`,
|
||||
which was never listed in requirements.in. Locally and in CI that import
|
||||
resolved anyway, because pytest depends on packaging -- so all 1159 tests
|
||||
passed. The Windows installer builds its venv from requirements.txt alone, with
|
||||
no test dependencies, so `import shopdb` raised ModuleNotFoundError and the
|
||||
whole application failed to start on a customer server.
|
||||
|
||||
Test dependencies are exactly the blind spot: they are present everywhere the
|
||||
test suite runs and absent everywhere it does not.
|
||||
|
||||
The check is static (AST) rather than a real import, so it covers plugins that
|
||||
are not installed at this site and code paths no test exercises.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
try:
|
||||
from importlib.metadata import packages_distributions
|
||||
except ImportError: # pragma: no cover - Python < 3.10
|
||||
packages_distributions = None
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SCANNED = ('shopdb', 'plugins', 'scripts')
|
||||
|
||||
# First-party top-level names. Imports of these resolve from the source tree.
|
||||
FIRST_PARTY = {'shopdb', 'plugins', 'scripts', 'migrations', 'tests', 'wsgi', 'conftest'}
|
||||
|
||||
# Modules deliberately imported without being declared. Each needs a reason.
|
||||
ALLOWED_UNDECLARED = {
|
||||
# Provided by the interpreter's own bundled installer tooling, and only
|
||||
# touched by developer scripts that never run on a deployed site.
|
||||
'pip',
|
||||
'setuptools',
|
||||
'pkg_resources',
|
||||
}
|
||||
|
||||
|
||||
def _normalise(name):
|
||||
"""PEP 503 normalisation, so Flask_Migrate and flask-migrate compare equal."""
|
||||
return name.lower().replace('_', '-').replace('.', '-')
|
||||
|
||||
|
||||
def _declared_distributions():
|
||||
"""Distribution names pinned in requirements.txt."""
|
||||
text = (REPO / 'requirements.txt').read_text(encoding='utf-8')
|
||||
declared = set()
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or line.startswith('--'):
|
||||
continue
|
||||
# "name==1.2.3 ; marker \" -> "name"
|
||||
head = line.split('==', 1)[0].split(';', 1)[0].strip()
|
||||
if head:
|
||||
declared.add(_normalise(head))
|
||||
return declared
|
||||
|
||||
|
||||
def _iter_source_files():
|
||||
for top in SCANNED:
|
||||
root = REPO / top
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for path in root.rglob('*.py'):
|
||||
parts = set(path.parts)
|
||||
if {'node_modules', 'venv', '.venv', 'build'} & parts:
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def _imported_top_level_modules(path):
|
||||
"""Top-level module names imported unconditionally by `path`.
|
||||
|
||||
Imports nested inside a `try:` are skipped: that is how this codebase spells
|
||||
an optional dependency, and a missing one is handled rather than fatal.
|
||||
Imports inside functions are still counted -- a deferred import of an
|
||||
undeclared package fails just as hard, only later.
|
||||
"""
|
||||
try:
|
||||
tree = ast.parse(path.read_text(encoding='utf-8'), filename=str(path))
|
||||
except SyntaxError:
|
||||
pytest.fail('could not parse {}'.format(path.relative_to(REPO)))
|
||||
|
||||
guarded = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Try):
|
||||
for child in ast.walk(node):
|
||||
guarded.add(id(child))
|
||||
|
||||
found = set()
|
||||
for node in ast.walk(tree):
|
||||
if id(node) in guarded:
|
||||
continue
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
found.add(alias.name.split('.')[0])
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
# level > 0 is a relative (first-party) import.
|
||||
if node.level == 0 and node.module:
|
||||
found.add(node.module.split('.')[0])
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.skipif(packages_distributions is None, reason='needs Python 3.10+')
|
||||
def test_runtime_imports_are_declared_dependencies():
|
||||
declared = _declared_distributions()
|
||||
assert 'flask' in declared, 'requirements.txt did not parse as expected'
|
||||
|
||||
module_to_dists = packages_distributions()
|
||||
stdlib = getattr(sys, 'stdlib_module_names', frozenset())
|
||||
|
||||
offenders = {}
|
||||
for path in _iter_source_files():
|
||||
for module in _imported_top_level_modules(path):
|
||||
if module in FIRST_PARTY or module in stdlib or module in ALLOWED_UNDECLARED:
|
||||
continue
|
||||
if module.startswith('_'):
|
||||
continue
|
||||
dists = module_to_dists.get(module)
|
||||
if not dists:
|
||||
# Not installed in this environment, so its distribution cannot
|
||||
# be identified. Absence here is not evidence of a problem.
|
||||
continue
|
||||
if not any(_normalise(d) in declared for d in dists):
|
||||
rel = str(path.relative_to(REPO))
|
||||
offenders.setdefault(
|
||||
'{} (from {})'.format(module, '/'.join(sorted(dists))), []
|
||||
).append(rel)
|
||||
|
||||
if offenders:
|
||||
lines = []
|
||||
for module, files in sorted(offenders.items()):
|
||||
shown = sorted(files)[:3]
|
||||
more = '' if len(files) <= 3 else ' (+{} more)'.format(len(files) - 3)
|
||||
lines.append(' {}\n {}{}'.format(module, ', '.join(shown), more))
|
||||
pytest.fail(
|
||||
'Imported at runtime but not declared in requirements.txt.\n'
|
||||
'These resolve here only because a test dependency supplies them; a\n'
|
||||
'production venv has no test dependencies and will fail to import.\n'
|
||||
'Add them to requirements.in and recompile.\n\n' + '\n'.join(lines)
|
||||
)
|
||||
|
||||
|
||||
def test_packaging_is_declared():
|
||||
"""Regression: the specific import that broke the Windows install.
|
||||
|
||||
Kept separate from the sweep above because that one can only flag a module
|
||||
it can attribute to an installed distribution -- an environment without
|
||||
packaging would skip it silently, which is precisely the shipping case.
|
||||
"""
|
||||
assert 'packaging' in _declared_distributions(), (
|
||||
'shopdb/plugins/loader.py imports packaging at module scope. Without it '
|
||||
'declared, `import shopdb` fails on any venv built from requirements.txt.'
|
||||
)
|
||||
Reference in New Issue
Block a user