3606d8d696
An air-gapped site cannot be scanned from anywhere else, so when a CVE lands the only way to answer 'is that component here, and at what version' was to RDP in and go looking. The frontend was the real blind spot: nothing recorded which version of leaflet, dompurify, jspdf or html2canvas ends up inside the compiled SPA. scripts/generate_sbom.py emits CycloneDX 1.6 covering both ecosystems - every pin in requirements.txt with the sha256 the installer enforces, and every package in package-lock.json. Build-only npm packages are marked scope 'excluded' rather than dropped, so 'not here' stays distinguishable from 'not looked for'. Dependency edges are real: uv's '# via' comments give the Python graph and package-lock gives the npm one. Hand-rolled rather than cyclonedx-py plus cyclonedx-npm because both inputs are already pinned and committed - this is a format translation, not a scan - and because the build box may be a work PC with nothing but Python and Node. It is deterministic by construction: same inputs, byte-identical output, so regenerating does not churn. Staged into the application tree by both builders, so it installs onto the server with the app. shopdb-admin.ps1 verify reports it and searches it by component name, which is the question actually being asked. Packages appearing at several depths in package-lock (node_modules/vite and node_modules/vitest/node_modules/vite) are merged, and a copy reachable outside the dev tree makes the component count as shipped. Emitting both produced duplicate bom-refs, which CycloneDX forbids and scanners reject; getting the dev merge backwards would have hidden a shipped package from a CVE search. Not covered by bundle-lock.json on purpose: its provenance is git, not the third-party payload.
349 lines
13 KiB
Python
349 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Emit a CycloneDX 1.6 SBOM for this application: Python and npm together.
|
|
|
|
Why this is hand-rolled rather than cyclonedx-py plus cyclonedx-npm:
|
|
|
|
- It has to run on the machine that builds the installer, which may be a work
|
|
PC with nothing but Python and Node. Two more toolchains to install, keep
|
|
current and match versions across build boxes is a real cost for output this
|
|
small.
|
|
- Both inputs are already fully pinned and committed - requirements.txt with a
|
|
sha256 per package, package-lock.json with an integrity per package - so
|
|
there is nothing to resolve. This is a format translation, not a scan.
|
|
- It is deterministic by construction, which the packaged tools are not
|
|
without extra flags. Same inputs, byte-identical output.
|
|
|
|
WHAT IT COVERS
|
|
|
|
Python every pin in requirements.txt, with the sha256 the installer enforces.
|
|
Environment markers are ignored on purpose: a requirement guarded by
|
|
sys_platform == 'win32' still installs on the target, which is Windows.
|
|
npm every package in frontend/package-lock.json. Development-only packages
|
|
are included but marked scope 'excluded', because they do not ship
|
|
inside the compiled SPA - a reader answering "are we exposed" needs to
|
|
see the distinction, not a list with them silently missing.
|
|
|
|
Dependency edges are real, not flat: uv writes '# via <parent>' comments into
|
|
requirements.txt, and package-lock records each package's dependencies.
|
|
|
|
NOT COVERED: the application's own source, which git records, and the operating
|
|
system. See deploy/windows/installer/bundle-lock.json for the third-party
|
|
binaries (Python installer, IIS MSIs) that ship alongside this.
|
|
|
|
Usage:
|
|
generate_sbom.py <repo-root> [-o out.json] [--timestamp ISO8601] [--appversion X]
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import binascii
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
SPEC_VERSION = '1.6'
|
|
GENERATOR = 'scripts/generate_sbom.py'
|
|
# Fixed namespace so the same component set always yields the same serial number.
|
|
# A random UUID per run would make every SBOM differ from the last for no reason.
|
|
SERIAL_NAMESPACE = uuid.UUID('6f9d1a1e-2b3c-4d5e-8f90-a1b2c3d4e5f6')
|
|
|
|
|
|
def normalize(name):
|
|
return re.sub(r'[^A-Za-z0-9.]+', '_', name).lower()
|
|
|
|
|
|
def read_app_version(repo):
|
|
init = os.path.join(repo, 'shopdb', '__init__.py')
|
|
try:
|
|
with open(init) as fh:
|
|
match = re.search(r"^__version__\s*=\s*'([^']+)'", fh.read(), re.M)
|
|
if match:
|
|
return match.group(1)
|
|
except OSError:
|
|
pass
|
|
return '0.0.0'
|
|
|
|
|
|
def parse_requirements(path):
|
|
"""Pins, their sha256 list, and which package pulled each one in.
|
|
|
|
uv emits, per entry:
|
|
flask==3.1.3 \\
|
|
--hash=sha256:... \\
|
|
--hash=sha256:...
|
|
# via
|
|
# -r requirements.in
|
|
# flask-migrate
|
|
|
|
'-r <file>' means the site asked for it directly; anything else is a parent.
|
|
"""
|
|
components, order = {}, []
|
|
current = None
|
|
in_via = False
|
|
with open(path) as fh:
|
|
for raw in fh:
|
|
line = raw.rstrip('\n')
|
|
stripped = line.strip()
|
|
match = re.match(r'^([A-Za-z0-9._-]+)==([^\s;\\]+)', stripped)
|
|
if match and not stripped.startswith('#'):
|
|
current = normalize(match.group(1))
|
|
in_via = False
|
|
if current not in components:
|
|
order.append(current)
|
|
components[current] = {
|
|
'name': match.group(1), 'version': match.group(2),
|
|
'hashes': [], 'parents': [], 'direct': False,
|
|
}
|
|
continue
|
|
if current is None:
|
|
continue
|
|
hash_match = re.search(r'--hash=sha256:([a-f0-9]{64})', stripped)
|
|
if hash_match:
|
|
components[current]['hashes'].append(hash_match.group(1))
|
|
continue
|
|
if stripped.startswith('#'):
|
|
body = stripped.lstrip('#').strip()
|
|
if body == 'via':
|
|
in_via = True
|
|
continue
|
|
if body.startswith('via '):
|
|
in_via = True
|
|
body = body[4:].strip()
|
|
if in_via and body:
|
|
if body.startswith('-r ') or body.startswith('-c '):
|
|
components[current]['direct'] = True
|
|
else:
|
|
components[current]['parents'].append(normalize(body))
|
|
continue
|
|
in_via = False
|
|
return [components[key] for key in order]
|
|
|
|
|
|
def integrity_to_hash(integrity):
|
|
"""npm 'sha512-<base64>' becomes a CycloneDX hash entry."""
|
|
if not integrity or '-' not in integrity:
|
|
return None
|
|
algorithm, _, encoded = integrity.partition('-')
|
|
algorithms = {'sha512': 'SHA-512', 'sha256': 'SHA-256', 'sha1': 'SHA-1'}
|
|
if algorithm not in algorithms:
|
|
return None
|
|
try:
|
|
digest = base64.b64decode(encoded)
|
|
except (binascii.Error, ValueError):
|
|
return None
|
|
return {'alg': algorithms[algorithm], 'content': digest.hex()}
|
|
|
|
|
|
def parse_package_lock(path):
|
|
"""Every locked package, plus the names the project depends on DIRECTLY.
|
|
|
|
The direct set comes from the lockfile's root entry. Without it the SBOM
|
|
claimed the application depended directly on all ~110 shipped packages,
|
|
which flattens the graph and makes 'what did we choose' unanswerable.
|
|
"""
|
|
with open(path) as fh:
|
|
data = json.load(fh)
|
|
packages = data.get('packages', {})
|
|
root = packages.get('', {})
|
|
direct = sorted((root.get('dependencies') or {}).keys())
|
|
# Keyed by (name, version), because npm installs the same package at several
|
|
# depths: node_modules/vite AND node_modules/vitest/node_modules/vite are two
|
|
# entries for one component. Emitting both produced duplicate bom-refs, which
|
|
# CycloneDX forbids and scanners reject.
|
|
merged = {}
|
|
for key, meta in packages.items():
|
|
if not key.startswith('node_modules/'):
|
|
continue
|
|
# Nested paths (a/node_modules/b) name the package after the LAST segment.
|
|
name = key.split('node_modules/')[-1]
|
|
version = meta.get('version')
|
|
if not version:
|
|
continue
|
|
entry = merged.get((name, version))
|
|
if entry is None:
|
|
merged[(name, version)] = {
|
|
'name': name,
|
|
'version': version,
|
|
'dev': bool(meta.get('dev') or meta.get('devOptional')),
|
|
'license': meta.get('license'),
|
|
'hash': integrity_to_hash(meta.get('integrity')),
|
|
'deps': set((meta.get('dependencies') or {}).keys()),
|
|
}
|
|
continue
|
|
# One copy reachable outside the dev tree means the component ships, so
|
|
# 'dev' only survives while EVERY instance is dev. Getting this backwards
|
|
# would mark a shipped package build-only and hide it from a CVE search.
|
|
entry['dev'] = entry['dev'] and bool(meta.get('dev') or meta.get('devOptional'))
|
|
entry['license'] = entry['license'] or meta.get('license')
|
|
entry['hash'] = entry['hash'] or integrity_to_hash(meta.get('integrity'))
|
|
entry['deps'] |= set((meta.get('dependencies') or {}).keys())
|
|
|
|
out = []
|
|
for entry in merged.values():
|
|
entry['deps'] = sorted(entry['deps'])
|
|
out.append(entry)
|
|
out.sort(key=lambda component: (component['name'], component['version']))
|
|
return out, direct
|
|
|
|
|
|
def purl(ecosystem, name, version):
|
|
from urllib.parse import quote
|
|
return 'pkg:%s/%s@%s' % (ecosystem, quote(name, safe='@/'), quote(version, safe=''))
|
|
|
|
|
|
def build(repo, timestamp, app_version):
|
|
requirements = os.path.join(repo, 'requirements.txt')
|
|
lockfile = os.path.join(repo, 'frontend', 'package-lock.json')
|
|
|
|
components, dependencies = [], []
|
|
root_ref = 'shopdb-flask@%s' % app_version
|
|
root_deps = []
|
|
|
|
pips = parse_requirements(requirements) if os.path.exists(requirements) else []
|
|
by_normalized = {}
|
|
for pin in pips:
|
|
ref = purl('pypi', pin['name'], pin['version'])
|
|
by_normalized[normalize(pin['name'])] = ref
|
|
entry = {
|
|
'type': 'library',
|
|
'bom-ref': ref,
|
|
'name': pin['name'],
|
|
'version': pin['version'],
|
|
'purl': ref,
|
|
'scope': 'required',
|
|
'properties': [{'name': 'shopdb:ecosystem', 'value': 'python'}],
|
|
}
|
|
if pin['hashes']:
|
|
entry['hashes'] = [{'alg': 'SHA-256', 'content': value}
|
|
for value in sorted(pin['hashes'])]
|
|
components.append(entry)
|
|
|
|
for pin in pips:
|
|
ref = by_normalized[normalize(pin['name'])]
|
|
if pin['direct']:
|
|
root_deps.append(ref)
|
|
for parent in pin['parents']:
|
|
parent_ref = by_normalized.get(parent)
|
|
if parent_ref:
|
|
dependencies.append((parent_ref, ref))
|
|
|
|
npms, npm_direct = parse_package_lock(lockfile) if os.path.exists(lockfile) else ([], [])
|
|
npm_refs = {}
|
|
for package in npms:
|
|
ref = purl('npm', package['name'], package['version'])
|
|
npm_refs.setdefault(package['name'], ref)
|
|
entry = {
|
|
'type': 'library',
|
|
'bom-ref': ref,
|
|
'name': package['name'],
|
|
'version': package['version'],
|
|
'purl': ref,
|
|
# Development packages do not ship inside the compiled SPA. Recorded
|
|
# rather than dropped so the distinction is visible.
|
|
'scope': 'excluded' if package['dev'] else 'required',
|
|
'properties': [{'name': 'shopdb:ecosystem', 'value': 'npm'}],
|
|
}
|
|
if package['hash']:
|
|
entry['hashes'] = [package['hash']]
|
|
if isinstance(package['license'], str):
|
|
entry['licenses'] = [{'license': {'id': package['license']}}]
|
|
components.append(entry)
|
|
|
|
for name in npm_direct:
|
|
ref = npm_refs.get(name)
|
|
if ref:
|
|
root_deps.append(ref)
|
|
|
|
for package in npms:
|
|
ref = npm_refs.get(package['name'])
|
|
for name in package['deps']:
|
|
child = npm_refs.get(name)
|
|
if child and ref:
|
|
dependencies.append((ref, child))
|
|
|
|
graph = {}
|
|
for parent, child in dependencies:
|
|
graph.setdefault(parent, set()).add(child)
|
|
depends_on = [{'ref': root_ref, 'dependsOn': sorted(set(root_deps))}]
|
|
for ref in sorted(graph):
|
|
depends_on.append({'ref': ref, 'dependsOn': sorted(graph[ref])})
|
|
|
|
components.sort(key=lambda component: component['bom-ref'])
|
|
|
|
fingerprint = hashlib.sha256(
|
|
json.dumps([component['bom-ref'] for component in components],
|
|
sort_keys=True).encode()).hexdigest()
|
|
serial = uuid.uuid5(SERIAL_NAMESPACE, fingerprint)
|
|
|
|
return {
|
|
'bomFormat': 'CycloneDX',
|
|
'specVersion': SPEC_VERSION,
|
|
'serialNumber': 'urn:uuid:%s' % serial,
|
|
'version': 1,
|
|
'metadata': {
|
|
'timestamp': timestamp,
|
|
'tools': {'components': [{
|
|
'type': 'application', 'name': GENERATOR, 'version': app_version,
|
|
}]},
|
|
'authors': [{'name': 'GE Aerospace'}],
|
|
'supplier': {'name': 'GE Aerospace'},
|
|
'component': {
|
|
'type': 'application',
|
|
'bom-ref': root_ref,
|
|
'name': 'shopdb-flask',
|
|
'version': app_version,
|
|
'description': 'Asset management for the shop floor.',
|
|
},
|
|
},
|
|
'components': components,
|
|
'dependencies': depends_on,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('repo')
|
|
parser.add_argument('-o', '--output', default='-')
|
|
# Pinned by the caller, or by SOURCE_DATE_EPOCH, so two builds of one commit
|
|
# produce byte-identical files.
|
|
parser.add_argument('--timestamp', default=None)
|
|
parser.add_argument('--appversion', default=None)
|
|
args = parser.parse_args()
|
|
|
|
timestamp = args.timestamp
|
|
if not timestamp:
|
|
epoch = os.environ.get('SOURCE_DATE_EPOCH')
|
|
moment = (datetime.fromtimestamp(int(epoch), timezone.utc) if epoch
|
|
else datetime.now(timezone.utc))
|
|
timestamp = moment.strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
version = args.appversion or read_app_version(args.repo)
|
|
document = build(args.repo, timestamp, version)
|
|
text = json.dumps(document, indent=2, sort_keys=False) + '\n'
|
|
|
|
if args.output == '-':
|
|
sys.stdout.write(text)
|
|
else:
|
|
directory = os.path.dirname(args.output)
|
|
if directory and not os.path.isdir(directory):
|
|
os.makedirs(directory)
|
|
with open(args.output, 'w') as fh:
|
|
fh.write(text)
|
|
counts = {}
|
|
for component in document['components']:
|
|
key = component['properties'][0]['value']
|
|
counts[key] = counts.get(key, 0) + 1
|
|
shipped = sum(1 for component in document['components']
|
|
if component['scope'] == 'required')
|
|
print('%s: %d components (%s), %d shipped' % (
|
|
args.output, len(document['components']),
|
|
', '.join('%s %d' % (k, counts[k]) for k in sorted(counts)), shipped))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|