diff --git a/deploy/windows/installer/README.md b/deploy/windows/installer/README.md index 0f29974..4344b51 100644 --- a/deploy/windows/installer/README.md +++ b/deploy/windows/installer/README.md @@ -139,6 +139,32 @@ For the West Jefferson production server specifically, this is a **migration, no an upgrade** - prod runs Python 3.13 against a hand-built deployment, so it needs a deliberate window, a database backup, and web.config reconciled by hand. +## Bill of materials + +Every build stages a CycloneDX 1.6 SBOM at `sbom.cdx.json`, inside the +application tree, so it installs onto the server with the app. Both ecosystems, +in one document: + +- **Python** — every pin in `requirements.txt`, with the sha256 the installer + enforces. Environment markers are ignored: a `sys_platform == 'win32'` + dependency still installs on the target. +- **npm** — every package in `frontend/package-lock.json`. Build-only packages + are marked `scope: excluded` rather than dropped, so "not here" is + distinguishable from "not looked for". + +It ships to the server because an air-gapped site cannot be scanned from +anywhere else. When a CVE lands, the answer is already on the box: + +```powershell +shopdb-admin.ps1 verify # counts, and which bundle this is +shopdb-admin.ps1 verify -Path leaflet # is that component here, at what version +``` + +Generated by `scripts/generate_sbom.py` from files that are already pinned and +committed, so it is a translation rather than a scan — no network, no extra +toolchain on the build box, and byte-identical output for the same inputs. It is +deliberately not in `bundle-lock.json`: its provenance is git, not the payload. + ## Client IP addresses IIS does not set `X-Forwarded-For` on its own, and HttpPlatformHandler connects diff --git a/deploy/windows/installer/build-installer.ps1 b/deploy/windows/installer/build-installer.ps1 index d585dbd..ccef0db 100644 --- a/deploy/windows/installer/build-installer.ps1 +++ b/deploy/windows/installer/build-installer.ps1 @@ -166,6 +166,15 @@ if (Test-Path $cfgSrc) { Copy-Item $cfgSrc (Join-Path $AppOut 'deploy\windows') -Force } +# A CycloneDX SBOM of everything this tree depends on, Python and npm together. +# Staged INTO the tree so it installs onto the server with the application: an +# air-gapped site cannot be scanned remotely, so the only way to answer "are we +# exposed to this CVE, and where" is for the answer to be sitting on the box. +Step 'Generating SBOM' +& $python.Source (Join-Path $RepoRoot 'scripts\generate_sbom.py') $RepoRoot ` + -o (Join-Path $AppOut 'sbom.cdx.json') | ForEach-Object { Say " $_" 'White' } +if ($LASTEXITCODE -ne 0) { Die 'SBOM generation failed' } + # Stage the profile INTO the tree: `flask plugin apply-profile` at provisioning # reads the same profile the tree was staged from, so the installed plugin set # and the shipped plugin code cannot drift. diff --git a/deploy/windows/installer/shopdb-admin.ps1 b/deploy/windows/installer/shopdb-admin.ps1 index cb486ea..03d783b 100644 --- a/deploy/windows/installer/shopdb-admin.ps1 +++ b/deploy/windows/installer/shopdb-admin.ps1 @@ -496,6 +496,39 @@ function Invoke-Verify { Say ' no bundle-lock.json recorded (installed before payload locking, or by hand)' 'Yellow' } + # The SBOM travels with the application, because a server on a vaulted + # network cannot be scanned from anywhere else. When a CVE lands, this is + # what answers "is that component here, and at what version" without needing + # the build box, the internet, or anyone's memory. + $sbom = Join-Path $AppRoot 'sbom.cdx.json' + if (Test-Path $sbom) { + try { + $b = Get-Content $sbom -Raw | ConvertFrom-Json + $shipped = @($b.components | Where-Object { $_.scope -eq 'required' }).Count + Say (" components : {0} ({1} shipped), CycloneDX {2}" -f ` + $b.components.Count, $shipped, $b.specVersion) + Say (" bill of materials: {0}" -f $sbom) 'DarkGray' + Say ' search it with : shopdb-admin.ps1 verify -Path ' 'DarkGray' + } catch { Say ' sbom.cdx.json is present but unreadable' 'Yellow' } + + # A named component turns this into the actual question being asked. + # Guarded on $b: an unreadable SBOM leaves it unset, and querying it then + # would report 'none', which reads as "you are not affected". + if ($Path -and $b) { + Say '' + Say (" matches for '{0}':" -f $Path) 'Cyan' + $hits = @($b.components | Where-Object { $_.name -like ('*' + $Path + '*') }) + if (-not $hits) { Say ' none - this server does not carry it' 'Green' } + foreach ($h in $hits) { + $tag = if ($h.scope -eq 'required') { 'SHIPPED' } else { 'build only' } + Say (" {0,-40} {1,-14} {2}" -f $h.name, $h.version, $tag) ` + $(if ($h.scope -eq 'required') { 'Yellow' } else { 'DarkGray' }) + } + } + } else { + Say ' no SBOM recorded (installed before SBOMs shipped, or by hand)' 'Yellow' + } + # pip's own audit. It re-reads the metadata of what is actually installed and # reports anything missing or version-inconsistent, which is the part that # can still drift after install - a hand-run `pip install` on the server. diff --git a/scripts/build-site.sh b/scripts/build-site.sh index 4d80689..7d3b4c1 100755 --- a/scripts/build-site.sh +++ b/scripts/build-site.sh @@ -77,6 +77,15 @@ if [ -f "$REPO/deploy/windows/web.config" ]; then cp -a "$REPO/deploy/windows/web.config" "$OUT/deploy/windows/" fi +# A CycloneDX SBOM of everything this tree depends on, Python and npm together. +# Staged INTO the tree so it installs onto the server with the application: an +# air-gapped site cannot be scanned remotely, so the only way to answer "are we +# exposed to this CVE, and where" is for the answer to be sitting on the box. +# Generated from requirements.txt and package-lock.json, both already pinned and +# committed, so it is a translation rather than a scan - and deterministic. +echo "==> Generating SBOM ..." +python3 "$REPO/scripts/generate_sbom.py" "$REPO" -o "$OUT/sbom.cdx.json" + # Stage the profile INTO the tree. This is what makes the set self-describing: # `flask plugin apply-profile` at provisioning time reads the same profile the # tree was staged from, so the installed plugin set and the shipped plugin code diff --git a/scripts/generate_sbom.py b/scripts/generate_sbom.py new file mode 100644 index 0000000..2501f1e --- /dev/null +++ b/scripts/generate_sbom.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +"""Emit a CycloneDX 1.6 SBOM for this application: Python and npm together. + +Why this is hand-rolled rather than cyclonedx-py plus cyclonedx-npm: + + - It has to run on the machine that builds the installer, which may be a work + PC with nothing but Python and Node. Two more toolchains to install, keep + current and match versions across build boxes is a real cost for output this + small. + - Both inputs are already fully pinned and committed - requirements.txt with a + sha256 per package, package-lock.json with an integrity per package - so + there is nothing to resolve. This is a format translation, not a scan. + - It is deterministic by construction, which the packaged tools are not + without extra flags. Same inputs, byte-identical output. + +WHAT IT COVERS + + Python every pin in requirements.txt, with the sha256 the installer enforces. + Environment markers are ignored on purpose: a requirement guarded by + sys_platform == 'win32' still installs on the target, which is Windows. + npm every package in frontend/package-lock.json. Development-only packages + are included but marked scope 'excluded', because they do not ship + inside the compiled SPA - a reader answering "are we exposed" needs to + see the distinction, not a list with them silently missing. + +Dependency edges are real, not flat: uv writes '# via ' comments into +requirements.txt, and package-lock records each package's dependencies. + +NOT COVERED: the application's own source, which git records, and the operating +system. See deploy/windows/installer/bundle-lock.json for the third-party +binaries (Python installer, IIS MSIs) that ship alongside this. + +Usage: + generate_sbom.py [-o out.json] [--timestamp ISO8601] [--appversion X] +""" +import argparse +import base64 +import binascii +import hashlib +import json +import os +import re +import sys +import uuid +from datetime import datetime, timezone + +SPEC_VERSION = '1.6' +GENERATOR = 'scripts/generate_sbom.py' +# Fixed namespace so the same component set always yields the same serial number. +# A random UUID per run would make every SBOM differ from the last for no reason. +SERIAL_NAMESPACE = uuid.UUID('6f9d1a1e-2b3c-4d5e-8f90-a1b2c3d4e5f6') + + +def normalize(name): + return re.sub(r'[^A-Za-z0-9.]+', '_', name).lower() + + +def read_app_version(repo): + init = os.path.join(repo, 'shopdb', '__init__.py') + try: + with open(init) as fh: + match = re.search(r"^__version__\s*=\s*'([^']+)'", fh.read(), re.M) + if match: + return match.group(1) + except OSError: + pass + return '0.0.0' + + +def parse_requirements(path): + """Pins, their sha256 list, and which package pulled each one in. + + uv emits, per entry: + flask==3.1.3 \\ + --hash=sha256:... \\ + --hash=sha256:... + # via + # -r requirements.in + # flask-migrate + + '-r ' means the site asked for it directly; anything else is a parent. + """ + components, order = {}, [] + current = None + in_via = False + with open(path) as fh: + for raw in fh: + line = raw.rstrip('\n') + stripped = line.strip() + match = re.match(r'^([A-Za-z0-9._-]+)==([^\s;\\]+)', stripped) + if match and not stripped.startswith('#'): + current = normalize(match.group(1)) + in_via = False + if current not in components: + order.append(current) + components[current] = { + 'name': match.group(1), 'version': match.group(2), + 'hashes': [], 'parents': [], 'direct': False, + } + continue + if current is None: + continue + hash_match = re.search(r'--hash=sha256:([a-f0-9]{64})', stripped) + if hash_match: + components[current]['hashes'].append(hash_match.group(1)) + continue + if stripped.startswith('#'): + body = stripped.lstrip('#').strip() + if body == 'via': + in_via = True + continue + if body.startswith('via '): + in_via = True + body = body[4:].strip() + if in_via and body: + if body.startswith('-r ') or body.startswith('-c '): + components[current]['direct'] = True + else: + components[current]['parents'].append(normalize(body)) + continue + in_via = False + return [components[key] for key in order] + + +def integrity_to_hash(integrity): + """npm 'sha512-' becomes a CycloneDX hash entry.""" + if not integrity or '-' not in integrity: + return None + algorithm, _, encoded = integrity.partition('-') + algorithms = {'sha512': 'SHA-512', 'sha256': 'SHA-256', 'sha1': 'SHA-1'} + if algorithm not in algorithms: + return None + try: + digest = base64.b64decode(encoded) + except (binascii.Error, ValueError): + return None + return {'alg': algorithms[algorithm], 'content': digest.hex()} + + +def parse_package_lock(path): + """Every locked package, plus the names the project depends on DIRECTLY. + + The direct set comes from the lockfile's root entry. Without it the SBOM + claimed the application depended directly on all ~110 shipped packages, + which flattens the graph and makes 'what did we choose' unanswerable. + """ + with open(path) as fh: + data = json.load(fh) + packages = data.get('packages', {}) + root = packages.get('', {}) + direct = sorted((root.get('dependencies') or {}).keys()) + # Keyed by (name, version), because npm installs the same package at several + # depths: node_modules/vite AND node_modules/vitest/node_modules/vite are two + # entries for one component. Emitting both produced duplicate bom-refs, which + # CycloneDX forbids and scanners reject. + merged = {} + for key, meta in packages.items(): + if not key.startswith('node_modules/'): + continue + # Nested paths (a/node_modules/b) name the package after the LAST segment. + name = key.split('node_modules/')[-1] + version = meta.get('version') + if not version: + continue + entry = merged.get((name, version)) + if entry is None: + merged[(name, version)] = { + 'name': name, + 'version': version, + 'dev': bool(meta.get('dev') or meta.get('devOptional')), + 'license': meta.get('license'), + 'hash': integrity_to_hash(meta.get('integrity')), + 'deps': set((meta.get('dependencies') or {}).keys()), + } + continue + # One copy reachable outside the dev tree means the component ships, so + # 'dev' only survives while EVERY instance is dev. Getting this backwards + # would mark a shipped package build-only and hide it from a CVE search. + entry['dev'] = entry['dev'] and bool(meta.get('dev') or meta.get('devOptional')) + entry['license'] = entry['license'] or meta.get('license') + entry['hash'] = entry['hash'] or integrity_to_hash(meta.get('integrity')) + entry['deps'] |= set((meta.get('dependencies') or {}).keys()) + + out = [] + for entry in merged.values(): + entry['deps'] = sorted(entry['deps']) + out.append(entry) + out.sort(key=lambda component: (component['name'], component['version'])) + return out, direct + + +def purl(ecosystem, name, version): + from urllib.parse import quote + return 'pkg:%s/%s@%s' % (ecosystem, quote(name, safe='@/'), quote(version, safe='')) + + +def build(repo, timestamp, app_version): + requirements = os.path.join(repo, 'requirements.txt') + lockfile = os.path.join(repo, 'frontend', 'package-lock.json') + + components, dependencies = [], [] + root_ref = 'shopdb-flask@%s' % app_version + root_deps = [] + + pips = parse_requirements(requirements) if os.path.exists(requirements) else [] + by_normalized = {} + for pin in pips: + ref = purl('pypi', pin['name'], pin['version']) + by_normalized[normalize(pin['name'])] = ref + entry = { + 'type': 'library', + 'bom-ref': ref, + 'name': pin['name'], + 'version': pin['version'], + 'purl': ref, + 'scope': 'required', + 'properties': [{'name': 'shopdb:ecosystem', 'value': 'python'}], + } + if pin['hashes']: + entry['hashes'] = [{'alg': 'SHA-256', 'content': value} + for value in sorted(pin['hashes'])] + components.append(entry) + + for pin in pips: + ref = by_normalized[normalize(pin['name'])] + if pin['direct']: + root_deps.append(ref) + for parent in pin['parents']: + parent_ref = by_normalized.get(parent) + if parent_ref: + dependencies.append((parent_ref, ref)) + + npms, npm_direct = parse_package_lock(lockfile) if os.path.exists(lockfile) else ([], []) + npm_refs = {} + for package in npms: + ref = purl('npm', package['name'], package['version']) + npm_refs.setdefault(package['name'], ref) + entry = { + 'type': 'library', + 'bom-ref': ref, + 'name': package['name'], + 'version': package['version'], + 'purl': ref, + # Development packages do not ship inside the compiled SPA. Recorded + # rather than dropped so the distinction is visible. + 'scope': 'excluded' if package['dev'] else 'required', + 'properties': [{'name': 'shopdb:ecosystem', 'value': 'npm'}], + } + if package['hash']: + entry['hashes'] = [package['hash']] + if isinstance(package['license'], str): + entry['licenses'] = [{'license': {'id': package['license']}}] + components.append(entry) + + for name in npm_direct: + ref = npm_refs.get(name) + if ref: + root_deps.append(ref) + + for package in npms: + ref = npm_refs.get(package['name']) + for name in package['deps']: + child = npm_refs.get(name) + if child and ref: + dependencies.append((ref, child)) + + graph = {} + for parent, child in dependencies: + graph.setdefault(parent, set()).add(child) + depends_on = [{'ref': root_ref, 'dependsOn': sorted(set(root_deps))}] + for ref in sorted(graph): + depends_on.append({'ref': ref, 'dependsOn': sorted(graph[ref])}) + + components.sort(key=lambda component: component['bom-ref']) + + fingerprint = hashlib.sha256( + json.dumps([component['bom-ref'] for component in components], + sort_keys=True).encode()).hexdigest() + serial = uuid.uuid5(SERIAL_NAMESPACE, fingerprint) + + return { + 'bomFormat': 'CycloneDX', + 'specVersion': SPEC_VERSION, + 'serialNumber': 'urn:uuid:%s' % serial, + 'version': 1, + 'metadata': { + 'timestamp': timestamp, + 'tools': {'components': [{ + 'type': 'application', 'name': GENERATOR, 'version': app_version, + }]}, + 'authors': [{'name': 'GE Aerospace'}], + 'supplier': {'name': 'GE Aerospace'}, + 'component': { + 'type': 'application', + 'bom-ref': root_ref, + 'name': 'shopdb-flask', + 'version': app_version, + 'description': 'Asset management for the shop floor.', + }, + }, + 'components': components, + 'dependencies': depends_on, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('repo') + parser.add_argument('-o', '--output', default='-') + # Pinned by the caller, or by SOURCE_DATE_EPOCH, so two builds of one commit + # produce byte-identical files. + parser.add_argument('--timestamp', default=None) + parser.add_argument('--appversion', default=None) + args = parser.parse_args() + + timestamp = args.timestamp + if not timestamp: + epoch = os.environ.get('SOURCE_DATE_EPOCH') + moment = (datetime.fromtimestamp(int(epoch), timezone.utc) if epoch + else datetime.now(timezone.utc)) + timestamp = moment.strftime('%Y-%m-%dT%H:%M:%SZ') + + version = args.appversion or read_app_version(args.repo) + document = build(args.repo, timestamp, version) + text = json.dumps(document, indent=2, sort_keys=False) + '\n' + + if args.output == '-': + sys.stdout.write(text) + else: + directory = os.path.dirname(args.output) + if directory and not os.path.isdir(directory): + os.makedirs(directory) + with open(args.output, 'w') as fh: + fh.write(text) + counts = {} + for component in document['components']: + key = component['properties'][0]['value'] + counts[key] = counts.get(key, 0) + 1 + shipped = sum(1 for component in document['components'] + if component['scope'] == 'required') + print('%s: %d components (%s), %d shipped' % ( + args.output, len(document['components']), + ', '.join('%s %d' % (k, counts[k]) for k in sorted(counts)), shipped)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/test_sbom.py b/tests/test_sbom.py new file mode 100644 index 0000000..76a1748 --- /dev/null +++ b/tests/test_sbom.py @@ -0,0 +1,154 @@ +"""The generated SBOM: shape, completeness, and the properties it is relied on for. + +The point of shipping an SBOM to an air-gapped site is answering "are we exposed +to this CVE, and where" without scanning the box. That only works if the document +actually lists everything that ships, at the right versions, with identifiers a +scanner recognises. These tests hold it to that. +""" +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] +GENERATOR = REPO / 'scripts' / 'generate_sbom.py' +FIXED_TIMESTAMP = '2026-01-01T00:00:00Z' + + +def generate(tmp_path, name='sbom.cdx.json'): + out = tmp_path / name + result = subprocess.run( + [sys.executable, str(GENERATOR), str(REPO), '-o', str(out), + '--timestamp', FIXED_TIMESTAMP], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + return json.loads(out.read_text()), out + + +@pytest.fixture(scope='module') +def sbom(tmp_path_factory): + document, _ = generate(tmp_path_factory.mktemp('sbom')) + return document + + +def test_is_valid_cyclonedx(sbom): + assert sbom['bomFormat'] == 'CycloneDX' + assert sbom['specVersion'] == '1.6' + assert sbom['serialNumber'].startswith('urn:uuid:') + assert sbom['version'] == 1 + + +def test_carries_the_ntia_minimum_elements(sbom): + """Supplier, component name, version, unique identifier, dependency + relationship, author, timestamp.""" + meta = sbom['metadata'] + assert meta['supplier']['name'] + assert meta['authors'] + assert meta['timestamp'] == FIXED_TIMESTAMP + assert meta['component']['name'] and meta['component']['version'] + assert sbom['dependencies'] + for component in sbom['components']: + assert component['name'] + assert component['version'] + assert component['purl'], '%s has no unique identifier' % component['name'] + + +def test_every_python_pin_is_present(sbom): + """The SBOM must not be quietly narrower than what the installer installs.""" + import re + pinned = set() + for line in (REPO / 'requirements.txt').read_text().splitlines(): + match = re.match(r'^([A-Za-z0-9._-]+)==([^\s;\\]+)', line.strip()) + if match: + pinned.add((match.group(1).lower().replace('_', '-'), match.group(2))) + listed = {(c['name'].lower().replace('_', '-'), c['version']) + for c in sbom['components'] if c['purl'].startswith('pkg:pypi/')} + assert pinned == listed + + +def test_marked_out_dependencies_are_still_listed(sbom): + """colorama is win32-only. It installs on the target, so it must appear - + the same blind spot that left it out of the wheelhouse.""" + names = {c['name'] for c in sbom['components'] if c['purl'].startswith('pkg:pypi/')} + assert 'colorama' in names + + +def test_frontend_packages_are_covered(sbom): + """The npm tree is the reason this exists: nothing else records what version + of leaflet or dompurify ends up inside the compiled SPA.""" + npm = {c['name']: c for c in sbom['components'] if c['purl'].startswith('pkg:npm/')} + assert len(npm) > 100 + for shipped in ('leaflet', 'dompurify', 'vue'): + assert shipped in npm, '%s is missing from the SBOM' % shipped + assert npm[shipped]['scope'] == 'required' + + +def test_build_only_packages_are_marked_not_dropped(sbom): + """Dev packages do not ship. Recorded as 'excluded' rather than omitted, so a + reader can tell 'not here' from 'not looked for'.""" + npm = {c['name']: c for c in sbom['components'] if c['purl'].startswith('pkg:npm/')} + assert npm['vite']['scope'] == 'excluded' + assert any(c['scope'] == 'required' for c in npm.values()) + + +def test_bom_refs_are_unique(sbom): + """CycloneDX forbids duplicate bom-refs, and scanners reject a document that + has them. npm installs the same package at several depths - node_modules/vite + and node_modules/vitest/node_modules/vite - which emitted it twice.""" + refs = [c['bom-ref'] for c in sbom['components']] + duplicates = {ref for ref in refs if refs.count(ref) > 1} + assert not duplicates, 'duplicate bom-refs: %s' % sorted(duplicates)[:5] + + +def test_a_package_present_outside_the_dev_tree_counts_as_shipped(tmp_path): + """Merging duplicates must not mark a shipped package build-only, which would + hide it from a CVE search on the server.""" + sys.path.insert(0, str(REPO / 'scripts')) + import generate_sbom + + lock = tmp_path / 'package-lock.json' + lock.write_text(json.dumps({'lockfileVersion': 3, 'packages': { + '': {'name': 'x', 'version': '1.0.0', 'dependencies': {'shared': '^1'}}, + 'node_modules/shared': {'version': '1.0.0'}, + 'node_modules/builder': {'version': '2.0.0', 'dev': True}, + 'node_modules/builder/node_modules/shared': {'version': '1.0.0', 'dev': True}, + }})) + packages, direct = generate_sbom.parse_package_lock(str(lock)) + shared = [p for p in packages if p['name'] == 'shared'] + assert len(shared) == 1, 'the two copies of shared were not merged' + assert shared[0]['dev'] is False + assert direct == ['shared'] + + +def test_components_carry_integrity_hashes(sbom): + missing = [c['name'] for c in sbom['components'] if not c.get('hashes')] + assert not missing, 'components without a hash: %s' % missing[:5] + + +def test_dependency_graph_is_real_not_flat(sbom): + """A flat 'root depends on everything' graph cannot answer what was chosen + versus what was dragged in.""" + root = sbom['metadata']['component']['bom-ref'] + edges = {entry['ref']: entry['dependsOn'] for entry in sbom['dependencies']} + assert root in edges + assert len(edges) > 1, 'no edges below the root' + assert len(edges[root]) < len(sbom['components']), 'root depends on everything' + + refs = {c['bom-ref'] for c in sbom['components']} + for ref, children in edges.items(): + if ref == root: + continue + assert ref in refs, 'edge from an unknown component: %s' % ref + for child in children: + assert child in refs, 'edge to an unknown component: %s' % child + + +def test_output_is_byte_identical_across_runs(tmp_path): + """Regenerating must not churn. A document that differs every build gets + re-committed without being read.""" + first, first_path = generate(tmp_path, 'a.json') + _, second_path = generate(tmp_path, 'b.json') + assert first_path.read_bytes() == second_path.read_bytes() + assert first['serialNumber'] == json.loads(second_path.read_text())['serialNumber']