shopdb-flask/scripts/generate_sbom.py

#!/usr/bin/env python3
"""Emit a CycloneDX 1.6 SBOM for this application: Python and npm together.

Why this is hand-rolled rather than cyclonedx-py plus cyclonedx-npm:

  - It has to run on the machine that builds the installer, which may be a work
    PC with nothing but Python and Node. Two more toolchains to install, keep
    current and match versions across build boxes is a real cost for output this
    small.
  - Both inputs are already fully pinned and committed - requirements.txt with a
    sha256 per package, package-lock.json with an integrity per package - so
    there is nothing to resolve. This is a format translation, not a scan.
  - It is deterministic by construction, which the packaged tools are not
    without extra flags. Same inputs, byte-identical output.

WHAT IT COVERS

  Python  every pin in requirements.txt, with the sha256 the installer enforces.
          Environment markers are ignored on purpose: a requirement guarded by
          sys_platform == 'win32' still installs on the target, which is Windows.
  npm     every package in frontend/package-lock.json. Development-only packages
          are included but marked scope 'excluded', because they do not ship
          inside the compiled SPA - a reader answering "are we exposed" needs to
          see the distinction, not a list with them silently missing.

Dependency edges are real, not flat: uv writes '# via <parent>' comments into
requirements.txt, and package-lock records each package's dependencies.

NOT COVERED: the application's own source, which git records, and the operating
system. A deployment that ships third-party binaries alongside the application
is responsible for recording those separately.

Usage:
  generate_sbom.py <repo-root> [-o out.json] [--timestamp ISO8601] [--appversion X]
"""
import argparse
import base64
import binascii
import hashlib
import json
import os
import re
import sys
import uuid
from datetime import datetime, timezone

SPEC_VERSION = '1.6'
GENERATOR = 'scripts/generate_sbom.py'
# Fixed namespace so the same component set always yields the same serial number.
# A random UUID per run would make every SBOM differ from the last for no reason.
SERIAL_NAMESPACE = uuid.UUID('6f9d1a1e-2b3c-4d5e-8f90-a1b2c3d4e5f6')


def normalize(name):
    return re.sub(r'[^A-Za-z0-9.]+', '_', name).lower()


def read_app_version(repo):
    init = os.path.join(repo, 'shopdb', '__init__.py')
    try:
        with open(init) as fh:
            match = re.search(r"^__version__\s*=\s*'([^']+)'", fh.read(), re.M)
        if match:
            return match.group(1)
    except OSError:
        pass
    return '0.0.0'


def parse_requirements(path):
    """Pins, their sha256 list, and which package pulled each one in.

    uv emits, per entry:
        flask==3.1.3 \\
            --hash=sha256:... \\
            --hash=sha256:...
            # via
            #   -r requirements.in
            #   flask-migrate

    '-r <file>' means the site asked for it directly; anything else is a parent.
    """
    components, order = {}, []
    current = None
    in_via = False
    with open(path) as fh:
        for raw in fh:
            line = raw.rstrip('\n')
            stripped = line.strip()
            match = re.match(r'^([A-Za-z0-9._-]+)==([^\s;\\]+)', stripped)
            if match and not stripped.startswith('#'):
                current = normalize(match.group(1))
                in_via = False
                if current not in components:
                    order.append(current)
                    components[current] = {
                        'name': match.group(1), 'version': match.group(2),
                        'hashes': [], 'parents': [], 'direct': False,
                    }
                continue
            if current is None:
                continue
            hash_match = re.search(r'--hash=sha256:([a-f0-9]{64})', stripped)
            if hash_match:
                components[current]['hashes'].append(hash_match.group(1))
                continue
            if stripped.startswith('#'):
                body = stripped.lstrip('#').strip()
                if body == 'via':
                    in_via = True
                    continue
                if body.startswith('via '):
                    in_via = True
                    body = body[4:].strip()
                if in_via and body:
                    if body.startswith('-r ') or body.startswith('-c '):
                        components[current]['direct'] = True
                    else:
                        components[current]['parents'].append(normalize(body))
                continue
            in_via = False
    return [components[key] for key in order]


def integrity_to_hash(integrity):
    """npm 'sha512-<base64>' becomes a CycloneDX hash entry."""
    if not integrity or '-' not in integrity:
        return None
    algorithm, _, encoded = integrity.partition('-')
    algorithms = {'sha512': 'SHA-512', 'sha256': 'SHA-256', 'sha1': 'SHA-1'}
    if algorithm not in algorithms:
        return None
    try:
        digest = base64.b64decode(encoded)
    except (binascii.Error, ValueError):
        return None
    return {'alg': algorithms[algorithm], 'content': digest.hex()}


def parse_package_lock(path):
    """Every locked package, plus the names the project depends on DIRECTLY.

    The direct set comes from the lockfile's root entry. Without it the SBOM
    claimed the application depended directly on all ~110 shipped packages,
    which flattens the graph and makes 'what did we choose' unanswerable.
    """
    with open(path) as fh:
        data = json.load(fh)
    packages = data.get('packages', {})
    root = packages.get('', {})
    direct = sorted((root.get('dependencies') or {}).keys())
    # Keyed by (name, version), because npm installs the same package at several
    # depths: node_modules/vite AND node_modules/vitest/node_modules/vite are two
    # entries for one component. Emitting both produced duplicate bom-refs, which
    # CycloneDX forbids and scanners reject.
    merged = {}
    for key, meta in packages.items():
        if not key.startswith('node_modules/'):
            continue
        # Nested paths (a/node_modules/b) name the package after the LAST segment.
        name = key.split('node_modules/')[-1]
        version = meta.get('version')
        if not version:
            continue
        entry = merged.get((name, version))
        if entry is None:
            merged[(name, version)] = {
                'name': name,
                'version': version,
                'dev': bool(meta.get('dev') or meta.get('devOptional')),
                'license': meta.get('license'),
                'hash': integrity_to_hash(meta.get('integrity')),
                'deps': set((meta.get('dependencies') or {}).keys()),
            }
            continue
        # One copy reachable outside the dev tree means the component ships, so
        # 'dev' only survives while EVERY instance is dev. Getting this backwards
        # would mark a shipped package build-only and hide it from a CVE search.
        entry['dev'] = entry['dev'] and bool(meta.get('dev') or meta.get('devOptional'))
        entry['license'] = entry['license'] or meta.get('license')
        entry['hash'] = entry['hash'] or integrity_to_hash(meta.get('integrity'))
        entry['deps'] |= set((meta.get('dependencies') or {}).keys())

    out = []
    for entry in merged.values():
        entry['deps'] = sorted(entry['deps'])
        out.append(entry)
    out.sort(key=lambda component: (component['name'], component['version']))
    return out, direct


def purl(ecosystem, name, version):
    from urllib.parse import quote
    return 'pkg:%s/%s@%s' % (ecosystem, quote(name, safe='@/'), quote(version, safe=''))


def build(repo, timestamp, app_version):
    requirements = os.path.join(repo, 'requirements.txt')
    lockfile = os.path.join(repo, 'frontend', 'package-lock.json')

    components, dependencies = [], []
    root_ref = 'shopdb-flask@%s' % app_version
    root_deps = []

    pips = parse_requirements(requirements) if os.path.exists(requirements) else []
    by_normalized = {}
    for pin in pips:
        ref = purl('pypi', pin['name'], pin['version'])
        by_normalized[normalize(pin['name'])] = ref
        entry = {
            'type': 'library',
            'bom-ref': ref,
            'name': pin['name'],
            'version': pin['version'],
            'purl': ref,
            'scope': 'required',
            'properties': [{'name': 'shopdb:ecosystem', 'value': 'python'}],
        }
        if pin['hashes']:
            entry['hashes'] = [{'alg': 'SHA-256', 'content': value}
                               for value in sorted(pin['hashes'])]
        components.append(entry)

    for pin in pips:
        ref = by_normalized[normalize(pin['name'])]
        if pin['direct']:
            root_deps.append(ref)
        for parent in pin['parents']:
            parent_ref = by_normalized.get(parent)
            if parent_ref:
                dependencies.append((parent_ref, ref))

    npms, npm_direct = parse_package_lock(lockfile) if os.path.exists(lockfile) else ([], [])
    npm_refs = {}
    for package in npms:
        ref = purl('npm', package['name'], package['version'])
        npm_refs.setdefault(package['name'], ref)
        entry = {
            'type': 'library',
            'bom-ref': ref,
            'name': package['name'],
            'version': package['version'],
            'purl': ref,
            # Development packages do not ship inside the compiled SPA. Recorded
            # rather than dropped so the distinction is visible.
            'scope': 'excluded' if package['dev'] else 'required',
            'properties': [{'name': 'shopdb:ecosystem', 'value': 'npm'}],
        }
        if package['hash']:
            entry['hashes'] = [package['hash']]
        if isinstance(package['license'], str):
            entry['licenses'] = [{'license': {'id': package['license']}}]
        components.append(entry)

    for name in npm_direct:
        ref = npm_refs.get(name)
        if ref:
            root_deps.append(ref)

    for package in npms:
        ref = npm_refs.get(package['name'])
        for name in package['deps']:
            child = npm_refs.get(name)
            if child and ref:
                dependencies.append((ref, child))

    graph = {}
    for parent, child in dependencies:
        graph.setdefault(parent, set()).add(child)
    depends_on = [{'ref': root_ref, 'dependsOn': sorted(set(root_deps))}]
    for ref in sorted(graph):
        depends_on.append({'ref': ref, 'dependsOn': sorted(graph[ref])})

    components.sort(key=lambda component: component['bom-ref'])

    fingerprint = hashlib.sha256(
        json.dumps([component['bom-ref'] for component in components],
                   sort_keys=True).encode()).hexdigest()
    serial = uuid.uuid5(SERIAL_NAMESPACE, fingerprint)

    return {
        'bomFormat': 'CycloneDX',
        'specVersion': SPEC_VERSION,
        'serialNumber': 'urn:uuid:%s' % serial,
        'version': 1,
        'metadata': {
            'timestamp': timestamp,
            'tools': {'components': [{
                'type': 'application', 'name': GENERATOR, 'version': app_version,
            }]},
            'authors': [{'name': 'GE Aerospace'}],
            'supplier': {'name': 'GE Aerospace'},
            'component': {
                'type': 'application',
                'bom-ref': root_ref,
                'name': 'shopdb-flask',
                'version': app_version,
                'description': 'Asset management for the shop floor.',
            },
        },
        'components': components,
        'dependencies': depends_on,
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('repo')
    parser.add_argument('-o', '--output', default='-')
    # Pinned by the caller, or by SOURCE_DATE_EPOCH, so two builds of one commit
    # produce byte-identical files.
    parser.add_argument('--timestamp', default=None)
    parser.add_argument('--appversion', default=None)
    args = parser.parse_args()

    timestamp = args.timestamp
    if not timestamp:
        epoch = os.environ.get('SOURCE_DATE_EPOCH')
        moment = (datetime.fromtimestamp(int(epoch), timezone.utc) if epoch
                  else datetime.now(timezone.utc))
        timestamp = moment.strftime('%Y-%m-%dT%H:%M:%SZ')

    version = args.appversion or read_app_version(args.repo)
    document = build(args.repo, timestamp, version)
    text = json.dumps(document, indent=2, sort_keys=False) + '\n'

    if args.output == '-':
        sys.stdout.write(text)
    else:
        directory = os.path.dirname(args.output)
        if directory and not os.path.isdir(directory):
            os.makedirs(directory)
        with open(args.output, 'w') as fh:
            fh.write(text)
        counts = {}
        for component in document['components']:
            key = component['properties'][0]['value']
            counts[key] = counts.get(key, 0) + 1
        shipped = sum(1 for component in document['components']
                      if component['scope'] == 'required')
        print('%s: %d components (%s), %d shipped' % (
            args.output, len(document['components']),
            ', '.join('%s %d' % (k, counts[k]) for k in sorted(counts)), shipped))
    return 0


if __name__ == '__main__':
    sys.exit(main())