Release 0.8.0: Windows installer, twelve fixes from a real Server 2019 install, operator docs

tests/test_runtime_dependencies.py

@@ -0,0 +1,159 @@
+"""Every third-party module the app imports at runtime must be a DECLARED dependency.
+
+The bug this exists to prevent: `shopdb/plugins/loader.py` imports `packaging`,
+which was never listed in requirements.in. Locally and in CI that import
+resolved anyway, because pytest depends on packaging -- so all 1159 tests
+passed. The Windows installer builds its venv from requirements.txt alone, with
+no test dependencies, so `import shopdb` raised ModuleNotFoundError and the
+whole application failed to start on a customer server.
+
+Test dependencies are exactly the blind spot: they are present everywhere the
+test suite runs and absent everywhere it does not.
+
+The check is static (AST) rather than a real import, so it covers plugins that
+are not installed at this site and code paths no test exercises.
+"""
+
+import ast
+import sys
+from pathlib import Path
+
+import pytest
+
+try:
+    from importlib.metadata import packages_distributions
+except ImportError:  # pragma: no cover - Python < 3.10
+    packages_distributions = None
+
+REPO = Path(__file__).resolve().parent.parent
+SCANNED = ('shopdb', 'plugins', 'scripts')
+
+# First-party top-level names. Imports of these resolve from the source tree.
+FIRST_PARTY = {'shopdb', 'plugins', 'scripts', 'migrations', 'tests', 'wsgi', 'conftest'}
+
+# Modules deliberately imported without being declared. Each needs a reason.
+ALLOWED_UNDECLARED = {
+    # Provided by the interpreter's own bundled installer tooling, and only
+    # touched by developer scripts that never run on a deployed site.
+    'pip',
+    'setuptools',
+    'pkg_resources',
+}
+
+
+def _normalise(name):
+    """PEP 503 normalisation, so Flask_Migrate and flask-migrate compare equal."""
+    return name.lower().replace('_', '-').replace('.', '-')
+
+
+def _declared_distributions():
+    """Distribution names pinned in requirements.txt."""
+    text = (REPO / 'requirements.txt').read_text(encoding='utf-8')
+    declared = set()
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith('#') or line.startswith('--'):
+            continue
+        # "name==1.2.3 ; marker \" -> "name"
+        head = line.split('==', 1)[0].split(';', 1)[0].strip()
+        if head:
+            declared.add(_normalise(head))
+    return declared
+
+
+def _iter_source_files():
+    for top in SCANNED:
+        root = REPO / top
+        if not root.is_dir():
+            continue
+        for path in root.rglob('*.py'):
+            parts = set(path.parts)
+            if {'node_modules', 'venv', '.venv', 'build'} & parts:
+                continue
+            yield path
+
+
+def _imported_top_level_modules(path):
+    """Top-level module names imported unconditionally by `path`.
+
+    Imports nested inside a `try:` are skipped: that is how this codebase spells
+    an optional dependency, and a missing one is handled rather than fatal.
+    Imports inside functions are still counted -- a deferred import of an
+    undeclared package fails just as hard, only later.
+    """
+    try:
+        tree = ast.parse(path.read_text(encoding='utf-8'), filename=str(path))
+    except SyntaxError:
+        pytest.fail('could not parse {}'.format(path.relative_to(REPO)))
+
+    guarded = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Try):
+            for child in ast.walk(node):
+                guarded.add(id(child))
+
+    found = set()
+    for node in ast.walk(tree):
+        if id(node) in guarded:
+            continue
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                found.add(alias.name.split('.')[0])
+        elif isinstance(node, ast.ImportFrom):
+            # level > 0 is a relative (first-party) import.
+            if node.level == 0 and node.module:
+                found.add(node.module.split('.')[0])
+    return found
+
+
+@pytest.mark.skipif(packages_distributions is None, reason='needs Python 3.10+')
+def test_runtime_imports_are_declared_dependencies():
+    declared = _declared_distributions()
+    assert 'flask' in declared, 'requirements.txt did not parse as expected'
+
+    module_to_dists = packages_distributions()
+    stdlib = getattr(sys, 'stdlib_module_names', frozenset())
+
+    offenders = {}
+    for path in _iter_source_files():
+        for module in _imported_top_level_modules(path):
+            if module in FIRST_PARTY or module in stdlib or module in ALLOWED_UNDECLARED:
+                continue
+            if module.startswith('_'):
+                continue
+            dists = module_to_dists.get(module)
+            if not dists:
+                # Not installed in this environment, so its distribution cannot
+                # be identified. Absence here is not evidence of a problem.
+                continue
+            if not any(_normalise(d) in declared for d in dists):
+                rel = str(path.relative_to(REPO))
+                offenders.setdefault(
+                    '{} (from {})'.format(module, '/'.join(sorted(dists))), []
+                ).append(rel)
+
+    if offenders:
+        lines = []
+        for module, files in sorted(offenders.items()):
+            shown = sorted(files)[:3]
+            more = '' if len(files) <= 3 else ' (+{} more)'.format(len(files) - 3)
+            lines.append('  {}\n      {}{}'.format(module, ', '.join(shown), more))
+        pytest.fail(
+            'Imported at runtime but not declared in requirements.txt.\n'
+            'These resolve here only because a test dependency supplies them; a\n'
+            'production venv has no test dependencies and will fail to import.\n'
+            'Add them to requirements.in and recompile.\n\n' + '\n'.join(lines)
+        )
+
+
+def test_packaging_is_declared():
+    """Regression: the specific import that broke the Windows install.
+
+    Kept separate from the sweep above because that one can only flag a module
+    it can attribute to an installed distribution -- an environment without
+    packaging would skip it silently, which is precisely the shipping case.
+    """
+    assert 'packaging' in _declared_distributions(), (
+        'shopdb/plugins/loader.py imports packaging at module scope. Without it '
+        'declared, `import shopdb` fails on any venv built from requirements.txt.'
+    )