From 13d831eb90505cdbc4daff811cd7ea5b0217c090 Mon Sep 17 00:00:00 2001 From: cproudlock Date: Mon, 3 Aug 2026 11:46:39 -0400 Subject: [PATCH] feat(installer): require the wheelhouse to satisfy requirements.txt, and lock the real payload The lock records what IS in the wheelhouse, not what the application NEEDS, so an incomplete wheelhouse was locked, blessed and shipped - and only failed on an air-gapped server. That is not hypothetical. Assembling the wheelhouse anywhere other than Windows silently omits colorama, a win32-only dependency of click, because pip evaluates environment markers against the machine doing the downloading rather than the machine being targeted. The bundle built here was short exactly that one wheel. Both verifiers now cross-check wheels/ against the staged requirements.txt, ignoring markers, since a requirement guarded by sys_platform == 'win32' is precisely the one that must be present. Names are normalised to PEP 427 wheel form, so mysql-connector-python matches mysql_connector_python. bundle-lock.json is the first real lock: 42 files, cp314/win_amd64 - 39 wheels, Python 3.14.6, HttpPlatformHandler 1.2 and URL Rewrite. MySQL is absent and optional; a site choosing the bundled-database option adds it and re-locks. The naming gate now skips the installer's build output. It contains a staged copy of the application plus a second SPA build under dist-subpath, which --exclude-dir=dist does not match, so a staged bundle failed the gate on vendored minified JS nobody in this repository wrote. --- deploy/windows/installer/bundle-lock.json | 197 ++++++++++++++++++ deploy/windows/installer/bundle-lock.ps1 | 40 ++++ .../windows/installer/verify_bundle_lock.py | 50 +++++ scripts/check-naming-and-style.sh | 9 +- tests/test_bundle_lock.py | 32 +++ 5 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 deploy/windows/installer/bundle-lock.json diff --git a/deploy/windows/installer/bundle-lock.json b/deploy/windows/installer/bundle-lock.json new file mode 100644 index 0000000..d56fef7 --- /dev/null +++ b/deploy/windows/installer/bundle-lock.json @@ -0,0 +1,197 @@ +{ + "schema": 1, + "generated": "2026-08-03T15:45:11Z", + "pythontag": "cp314", + "platform": "win_amd64", + "note": "Exact third-party payload of the installer bundle. Regenerate with refresh-bundle-lock.ps1 and COMMIT the change as a reviewed dependency bump.", + "payloads": { + "wheels": { + "required": true, + "files": { + "jinja2-3.1.6-py3-none-any.whl": { + "size": 134899, + "sha256": "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" + }, + "itsdangerous-2.2.0-py3-none-any.whl": { + "size": 16234, + "sha256": "c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef" + }, + "cffi-2.1.0-cp314-cp314-win_amd64.whl": { + "size": 187937, + "sha256": "1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb" + }, + "email_validator-2.3.0-py3-none-any.whl": { + "size": 35604, + "sha256": "80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4" + }, + "urllib3-2.7.0-py3-none-any.whl": { + "size": 131087, + "sha256": "9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" + }, + "flask_jwt_extended-4.7.3-py2.py3-none-any.whl": { + "size": 22698, + "sha256": "905ac807b52b5409bc9244dbcca434968c13ca6f9d91bffe7d4cb71e0e6231cb" + }, + "flask_sqlalchemy-3.1.1-py3-none-any.whl": { + "size": 25125, + "sha256": "4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0" + }, + "pycparser-3.0-py3-none-any.whl": { + "size": 48172, + "sha256": "b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" + }, + "requests-2.33.1-py3-none-any.whl": { + "size": 64947, + "sha256": "4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a" + }, + "blinker-1.9.0-py3-none-any.whl": { + "size": 8458, + "sha256": "ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc" + }, + "greenlet-3.5.0-cp314-cp314-win_amd64.whl": { + "size": 239835, + "sha256": "3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033" + }, + "marshmallow_sqlalchemy-1.5.0-py3-none-any.whl": { + "size": 16582, + "sha256": "3865232672f3dd38c4d5e4e85fdedce76904200742c3594948a2d11d0af93258" + }, + "alembic-1.18.4-py3-none-any.whl": { + "size": 263893, + "sha256": "a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a" + }, + "pyjwt-2.12.1-py3-none-any.whl": { + "size": 29726, + "sha256": "28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c" + }, + "mako-1.3.12-py3-none-any.whl": { + "size": 78521, + "sha256": "8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9" + }, + "waitress-3.0.2-py3-none-any.whl": { + "size": 56232, + "sha256": "c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e" + }, + "colorama-0.4.6-py2.py3-none-any.whl": { + "size": 25335, + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + }, + "certifi-2026.4.22-py3-none-any.whl": { + "size": 135707, + "sha256": "3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a" + }, + "cachelib-0.13.0-py3-none-any.whl": { + "size": 20914, + "sha256": "8c8019e53b6302967d4e8329a504acf75e7bc46130291d30188a6e4e58162516" + }, + "pymysql-1.1.3-py3-none-any.whl": { + "size": 45356, + "sha256": "8164ba62c552f6105f3b11753352d0f16b90d1703ba67d81923d5a8a5d1c5289" + }, + "werkzeug-3.1.8-py3-none-any.whl": { + "size": 226459, + "sha256": "63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50" + }, + "idna-3.13-py3-none-any.whl": { + "size": 68629, + "sha256": "892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3" + }, + "python_dotenv-1.2.2-py3-none-any.whl": { + "size": 22101, + "sha256": "1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a" + }, + "flask_marshmallow-1.5.0-py3-none-any.whl": { + "size": 12161, + "sha256": "99951c77e5654111ed733811c6dc9310bfb4c3688c78a9e76f80b5ae0b2279a6" + }, + "markupsafe-3.0.3-cp314-cp314-win_amd64.whl": { + "size": 15341, + "sha256": "bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581" + }, + "flask_cors-6.0.2-py3-none-any.whl": { + "size": 13257, + "sha256": "e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a" + }, + "flask_caching-2.4.0-py3-none-any.whl": { + "size": 28727, + "sha256": "d15b8135f055c4f28f6f7dbcf8d36a3de4af1224def975ae6e0b43cbfa684486" + }, + "tzdata-2026.3-py2.py3-none-any.whl": { + "size": 348168, + "sha256": "dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931" + }, + "charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl": { + "size": 159634, + "sha256": "92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f" + }, + "sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl": { + "size": 2144204, + "sha256": "77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5" + }, + "marshmallow-4.3.0-py3-none-any.whl": { + "size": 49148, + "sha256": "46c4fe6984707e3cbd485dfebbf0a59874f58d695aad05c1668d15e8c6e13b46" + }, + "dnspython-2.8.0-py3-none-any.whl": { + "size": 331094, + "sha256": "01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af" + }, + "typing_extensions-4.15.0-py3-none-any.whl": { + "size": 44614, + "sha256": "f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548" + }, + "flask-3.1.3-py3-none-any.whl": { + "size": 103424, + "sha256": "f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c" + }, + "tabulate-0.10.0-py3-none-any.whl": { + "size": 39814, + "sha256": "f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3" + }, + "mysql_connector_python-9.7.0-cp314-cp314-win_amd64.whl": { + "size": 18197576, + "sha256": "5a5abbc152bc28cb2e64a04605ecd9941eff6b0dc5f9528cb84adb873e9a1e49" + }, + "Flask_Migrate-4.1.0-py3-none-any.whl": { + "size": 21237, + "sha256": "24d8051af161782e0743af1b04a152d007bad9772b2bca67b7ec1e8ceeb3910d" + }, + "cryptography-50.0.0-cp311-abi3-win_amd64.whl": { + "size": 3840395, + "sha256": "bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30" + }, + "click-8.3.3-py3-none-any.whl": { + "size": 110502, + "sha256": "a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613" + } + } + }, + "python": { + "required": true, + "files": { + "python-3.14.6-amd64.exe": { + "size": 30774112, + "sha256": "14b3e9a710a3fcf0bd9b55ab6b60412bd91227563f813fc49040cabc0209e0bd" + } + } + }, + "urlrewrite": { + "required": false, + "files": { + "rewrite_amd64_en-US.msi": { + "size": 6078464, + "sha256": "37342ff2f585f263f34f48e9de59eb1051d61015a8e967dbde4075716230a32a" + } + } + }, + "httpplatformhandler": { + "required": true, + "files": { + "httpPlatformHandler_amd64.msi": { + "size": 557056, + "sha256": "90f8d4905a0ab4f2c95223b3c79e2807a0b74507747d240e43c4302e8db4b5ef" + } + } + } + } +} diff --git a/deploy/windows/installer/bundle-lock.ps1 b/deploy/windows/installer/bundle-lock.ps1 index 6695fcc..a1e862f 100644 --- a/deploy/windows/installer/bundle-lock.ps1 +++ b/deploy/windows/installer/bundle-lock.ps1 @@ -182,6 +182,46 @@ function Test-BundleLock { if (-not $expected.ContainsKey($rel)) { $problems += "$name/$rel is in the bundle but NOT in the lock (unexpected extra file)" } } } + $problems += Test-WheelhouseCoversRequirements -BundleRoot $BundleRoot + return $problems +} + +function Test-WheelhouseCoversRequirements { + <# + The lock records what IS in the wheelhouse, not what the application NEEDS. + Without this an incomplete wheelhouse gets locked, blessed, and shipped, + and the install fails on an air-gapped server. + + Not hypothetical: assembling the wheelhouse anywhere other than Windows + silently omits colorama, a win32-only dependency of click, because pip + evaluates environment markers against the machine doing the downloading + rather than the machine being targeted. Markers are therefore IGNORED here + - a requirement guarded by sys_platform == 'win32' is precisely the one + that has to be present. + #> + param([Parameter(Mandatory = $true)] [string] $BundleRoot) + $wheels = Join-Path $BundleRoot 'wheels' + $reqs = Join-Path $BundleRoot 'app\requirements.txt' + if (-not (Test-Path $wheels) -or -not (Test-Path $reqs)) { return @() } + + $have = @(Get-ChildItem $wheels -File -ErrorAction SilentlyContinue | ForEach-Object { $_.Name.ToLower() }) + $problems = @() + $pins = @{} + foreach ($line in (Get-Content $reqs)) { + $trimmed = $line.Trim() + if (-not $trimmed -or $trimmed.StartsWith('#')) { continue } + if ($trimmed -match '^([A-Za-z0-9._-]+)==([^\s;\\]+)') { + # PEP 427 wheel filename form: runs of non-alphanumerics become one _. + $pins[([regex]::Replace($Matches[1], '[^A-Za-z0-9.]+', '_')).ToLower()] = $Matches[2] + } + } + foreach ($name in ($pins.Keys | Sort-Object)) { + $prefix = "$name-$($pins[$name])-" + if (-not ($have | Where-Object { $_.StartsWith($prefix) })) { + $problems += ("wheels/ has no wheel for {0}=={1}, which requirements.txt pins " + + "(a marked-out dependency still installs on Windows)") -f $name, $pins[$name] + } + } return $problems } diff --git a/deploy/windows/installer/verify_bundle_lock.py b/deploy/windows/installer/verify_bundle_lock.py index 2c11c57..ad8fc2b 100644 --- a/deploy/windows/installer/verify_bundle_lock.py +++ b/deploy/windows/installer/verify_bundle_lock.py @@ -22,6 +22,7 @@ Usage: verify_bundle_lock.py import hashlib import json import os +import re import sys # Must match $script:BundlePayloads in bundle-lock.ps1. @@ -55,6 +56,54 @@ def payload_files(directory): return found +def normalize(name): + """PEP 427 wheel filename form: runs of non-alphanumerics become one _.""" + return re.sub(r'[^A-Za-z0-9.]+', '_', name).lower() + + +def requirement_pins(requirements_path): + """Every 'name==version' pinned in a lockfile, including marked-out ones. + + Markers are deliberately IGNORED. A requirement guarded by + sys_platform == 'win32' is exactly the case that must be present, because the + target is Windows and the wheelhouse is usually assembled somewhere else. + """ + pins = {} + with open(requirements_path) as fh: + for line in fh: + line = line.strip() + if not line or line.startswith('#'): + continue + match = re.match(r'^([A-Za-z0-9._-]+)==([^\s;\\]+)', line) + if match: + pins[normalize(match.group(1))] = match.group(2) + return pins + + +def check_wheelhouse_covers_requirements(bundle_root): + """The lock records what IS in the wheelhouse, not what the app NEEDS. + + Without this, an incomplete wheelhouse gets locked and blessed, and the + install fails on an air-gapped server. That is not hypothetical: assembling + the wheelhouse on Linux silently omits colorama, a win32-only dependency of + click, because pip evaluates environment markers against the machine doing + the downloading rather than the machine being targeted. + """ + wheels = os.path.join(bundle_root, 'wheels') + requirements = os.path.join(bundle_root, 'app', 'requirements.txt') + if not os.path.isdir(wheels) or not os.path.exists(requirements): + return [] + have = os.listdir(wheels) + problems = [] + for name, version in sorted(requirement_pins(requirements).items()): + prefix = '%s-%s-' % (name, version) + if not any(f.lower().startswith(prefix) for f in have): + problems.append( + 'wheels/ has no wheel for %s==%s, which requirements.txt pins ' + '(a marked-out dependency still installs on Windows)' % (name, version)) + return problems + + def verify(bundle_root, lock): problems = [] locked = lock.get('payloads') @@ -97,6 +146,7 @@ def verify(bundle_root, lock): problems.append( '%s/%s is in the bundle but NOT in the lock (unexpected extra file)' % (name, rel)) + problems.extend(check_wheelhouse_covers_requirements(bundle_root)) return problems diff --git a/scripts/check-naming-and-style.sh b/scripts/check-naming-and-style.sh index 9f2c59d..39aa0f5 100755 --- a/scripts/check-naming-and-style.sh +++ b/scripts/check-naming-and-style.sh @@ -9,7 +9,8 @@ # 4. No snake_case API params in frontend that should match DB column names # # Exits non-zero if any violation found. -# Skips: venv/, node_modules/, __pycache__/, frontend/dist/, migrations/versions/ +# Skips: venv/, node_modules/, __pycache__/, frontend/dist/, migrations/versions/, +# deploy/windows/installer/bundle/ (installer build output) set -e @@ -23,6 +24,12 @@ EXCLUDES=( --exclude-dir=node_modules --exclude-dir=__pycache__ --exclude-dir=dist + # The installer's build output: a staged copy of the whole application plus + # a second SPA build under dist-subpath, which --exclude-dir=dist does not + # match. Linting it means linting vendored minified JS and failing on + # characters nobody in this repository wrote. + --exclude-dir=bundle + --exclude-dir=dist-subpath --exclude-dir=.git --exclude-dir=versions --exclude-dir=staticdocs diff --git a/tests/test_bundle_lock.py b/tests/test_bundle_lock.py index 6b45f3a..0f8fa8b 100644 --- a/tests/test_bundle_lock.py +++ b/tests/test_bundle_lock.py @@ -185,6 +185,38 @@ def test_lock_missing_files_section_is_not_a_silent_pass(locked): assert all('wheels/' in p for p in problems) +def test_wheelhouse_must_cover_requirements(locked): + """The regression that motivated the check. + + A wheelhouse assembled anywhere but Windows omits colorama - pip evaluates + sys_platform markers against the downloading machine. The lock alone cannot + catch it, because the lock records what is there, not what is needed. + """ + bundle, lock = locked + app = bundle / 'app' + app.mkdir() + (app / 'requirements.txt').write_text( + "alembic==1.18.4 \\\n --hash=sha256:abc\n" + "colorama==0.4.6 ; sys_platform == 'win32' \\\n --hash=sha256:def\n") + problems = check_both(bundle, lock) + assert any('colorama==0.4.6' in p for p in problems), problems + assert not any('alembic' in p for p in problems), 'alembic has a wheel and must not be flagged' + + +def test_wheelhouse_coverage_normalizes_names(locked): + """mysql-connector-python pins as dashes and ships as mysql_connector_python.""" + bundle, lock = locked + app = bundle / 'app' + app.mkdir() + (app / 'requirements.txt').write_text('mysql-connector-python==9.7.0\n') + assert any('mysql_connector_python==9.7.0' in p for p in check_both(bundle, lock)) + + wheel = bundle / 'wheels' / 'mysql_connector_python-9.7.0-cp314-cp314-win_amd64.whl' + wheel.write_bytes(b'connector') + write_lock(bundle, lock) # re-lock so the new wheel is not an 'extra' + assert check_both(bundle, lock) == [] + + def test_payload_list_matches_powershell(): """Both files enumerate the payload directories. Same names, same required flags, same order - a directory that is required in one and optional in the