From 18f20b00e2c0cecfbb310f5cb0a8d221f12d3601 Mon Sep 17 00:00:00 2001 From: cproudlock Date: Thu, 15 Oct 2026 09:41:16 -0400 Subject: [PATCH] Offline bundle: pin and hash the webapp wheels, refuse unpinned ones, check for updates - webapp/requirements.txt pins all eight wheels with sha256, at the fixed releases (lxml 6.1.3 for GHSA-vfmq-68hx-4jfw / PYSEC-2026-87, werkzeug 3.1.9 for GHSA-g6x2-hccm-hh4m). - download-packages.sh fetches exactly that set for Python 3.12 / x86_64 into an emptied folder; PIP_DIR is honoured inside the container (own mount), files are handed back to the calling user, USE_DOCKER=1 forces the container on a noble host, an unknown package name now prints apt's error instead of failing silently, and linux-modules-extra-7.0.* is gone (it does not exist; 7.0 ships those modules in linux-modules). - Playbook: the wheel installer refuses any wheel whose sha256 is not pinned, needs exactly one wheel per pin, and removes an older installed version by its RECORD first. The webapp copy is a full rsync every run (was cp with creates: app.py, which never updated an install). - scripts/check-updates.py: offline debs against noble security/updates and the pins against PyPI + osv.dev; exit 1 when something needs updating. Run it before building a USB. Tested: installer in a Python 3.12 venv (clean install, upgrade over lxml 6.0.4 + werkzeug 3.1.8 leaves no old dist-info, re-run is a no-op, a stray wheel is refused); check-updates.py flags the April bundle (94 security updates) and passes the pins. --- playbook/pxe_server_setup.yml | 58 +++++++++++++--- scripts/check-updates.py | 123 ++++++++++++++++++++++++++++++++++ scripts/download-packages.sh | 39 ++++++++--- webapp/requirements.txt | 26 ++++++- 4 files changed, 225 insertions(+), 21 deletions(-) create mode 100755 scripts/check-updates.py diff --git a/playbook/pxe_server_setup.yml b/playbook/pxe_server_setup.yml index e09f40b..57724a8 100644 --- a/playbook/pxe_server_setup.yml +++ b/playbook/pxe_server_setup.yml @@ -1057,7 +1057,7 @@ files (vmlinuz-ubuntu-{{ blancco_ksuffix }}, kexec-initrd-{{ blancco_ksuffix }}.img, kmod-{{ blancco_kver }}.tar.gz) are NOT built. Install - linux-image-{{ blancco_kver }} + linux-modules-extra-{{ blancco_kver }} + linux-image-{{ blancco_kver }} + linux-modules-{{ blancco_kver }} (scripts/download-packages.sh, BLANCCO_KVER) or set blancco_kver to an installed version (ls /lib/modules), then re-run. when: not blancco_kernel_ok @@ -1445,10 +1445,15 @@ mode: '0755' - name: "Copy webapp from USB" - shell: > - cp -r "{{ usb_root }}/webapp/"* /opt/pxe-webapp/ 2>/dev/null || true + # The whole tree (app, config, services, views, templates, static), every + # run, so a re-run with a newer USB updates the app. Was: cp with + # creates: app.py, which never updated an existing install. + shell: | + set -e + test -f "{{ usb_root }}/webapp/app.py" + rsync -a --delete --exclude __pycache__ "{{ usb_root }}/webapp/" /opt/pxe-webapp/ args: - creates: /opt/pxe-webapp/app.py + executable: /bin/bash - name: "Install webapp Python dependencies (offline wheels)" args: @@ -1465,16 +1470,51 @@ if [ -n "$WHEEL_DIR" ]; then # Install wheels directly using Python zipfile (bypasses pip entirely) # This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12 + export REQS="{{ usb_root }}/webapp/requirements.txt" python3 << 'PYEOF' - import zipfile, sysconfig, glob, os + # Installs only the pinned wheels: the sha256 of every wheel must be in + # webapp/requirements.txt, and each pinned package must have exactly one + # wheel. An older installed version of a package is removed first (by its + # RECORD), so an upgrade does not leave the old files beside the new. + import zipfile, sysconfig, glob, os, re, sys, hashlib, csv, shutil site = sysconfig.get_path('platlib') wheel_dir = os.environ['WHEEL_DIR'] - for whl in sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))): - name = os.path.basename(whl) - print(f' Installing {name}') + reqs = open(os.environ['REQS']).read() + pins, hashes = {}, set() + for m in re.finditer(r'^([A-Za-z0-9_.-]+)==([^\s\\]+)((?:\s*\\?\s*--hash=sha256:[0-9a-f]{64})+)', reqs, re.M): + name = re.sub(r'[-_.]+', '_', m.group(1)).lower() + pins[name] = m.group(2) + hashes.update(re.findall(r'sha256:([0-9a-f]{64})', m.group(3))) + wheels = sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))) + seen = {} + for whl in wheels: + h = hashlib.sha256(open(whl, 'rb').read()).hexdigest() + name = re.sub(r'[-_.]+', '_', os.path.basename(whl).split('-')[0]).lower() + if h not in hashes or name not in pins: + sys.exit(f'REFUSED: {os.path.basename(whl)} is not a pinned wheel (sha256 {h[:16]}...)') + seen.setdefault(name, []).append(whl) + bad = [n for n in pins if len(seen.get(n, [])) != 1] + if bad: + sys.exit(f'REFUSED: need exactly one wheel for each pinned package; check: {", ".join(bad)}') + for name, (whl,) in sorted(seen.items()): + for old in glob.glob(os.path.join(site, '*.dist-info')): + stem = os.path.basename(old)[:-len('.dist-info')] + oname = re.sub(r'[-_.]+', '_', stem.rsplit('-', 1)[0]).lower() + if oname != name: + continue + rec = os.path.join(old, 'RECORD') + if os.path.isfile(rec): + for row in csv.reader(open(rec)): + if not row: + continue + f = os.path.normpath(os.path.join(site, row[0])) + if f.startswith(site + os.sep) and os.path.isfile(f): + os.remove(f) + shutil.rmtree(old, ignore_errors=True) + print(f' Installing {os.path.basename(whl)}') with zipfile.ZipFile(whl) as z: z.extractall(site) - print('All wheels installed to ' + site) + print('All pinned wheels installed to ' + site) PYEOF else # Fallback: try system pip (works if system has internet and compatible pip) diff --git a/scripts/check-updates.py b/scripts/check-updates.py new file mode 100755 index 0000000..29f7bf5 --- /dev/null +++ b/scripts/check-updates.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""check-updates.py - is the offline bundle current? Run before building a USB. + + * offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates, + noble-security; main + universe): which have a newer version, and which + of those are security updates. + * webapp/requirements.txt pins against PyPI (newer release?) and osv.dev + (known vulnerabilities in the pinned version). + +Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found +(the details are printed); 2 = could not check (no network, bad input). +Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only. + +Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all] +""" +import argparse +import glob +import gzip +import json +import os +import re +import subprocess +import sys +import urllib.request + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists" +SUITES = ("noble", "noble-updates", "noble-security") +COMPONENTS = ("main", "universe") + + +def fetch(url, timeout=60): + with urllib.request.urlopen(url, timeout=timeout) as r: + return r.read() + + +def newer(a, b): + """True if Debian version a > b.""" + return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0 + + +def ubuntu_index(): + """({package: newest version}, {package: newest security version}).""" + best, sec = {}, {} + for suite in SUITES: + for comp in COMPONENTS: + text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace") + pkg = None + for line in text.splitlines(): + if line.startswith("Package: "): + pkg = line[9:].strip() + elif line.startswith("Version: ") and pkg: + v = line[9:].strip() + if pkg not in best or newer(v, best[pkg]): + best[pkg] = v + if suite.endswith("security") and (pkg not in sec or newer(v, sec[pkg])): + sec[pkg] = v + return best, sec + + +def check_debs(deb_dir, show_all): + debs = glob.glob(os.path.join(deb_dir, "*.deb")) + if not debs: + print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)") + return 1 + best, sec = ubuntu_index() + stale, security = [], [] + for d in debs: + parts = os.path.basename(d)[:-4].split("_") + name, ver = parts[0], parts[1].replace("%3a", ":") + if name in best and newer(best[name], ver): + is_sec = name in sec and newer(sec[name], ver) + (security if is_sec else stale).append((name, ver, best[name])) + print(f"Ubuntu packages: {len(debs)} in {deb_dir}; {len(security) + len(stale)} have newer versions, " + f"{len(security)} of them security updates") + for n, v, nv in sorted(security): + print(f" SECURITY {n:34} {v:30} -> {nv}") + if show_all: + for n, v, nv in sorted(stale): + print(f" update {n:34} {v:30} -> {nv}") + return len(security) + + +def check_pins(reqs): + pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M) + if not pins: + print(f"no pinned packages in {reqs}") + return 0 + q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]} + req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(), + headers={"Content-Type": "application/json"}) + results = json.load(urllib.request.urlopen(req, timeout=60))["results"] + bad = 0 + print(f"Python pins in {os.path.relpath(reqs, ROOT)}:") + for (n, v), r in zip(pins, results): + try: + latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"] + except Exception: # report, keep going + latest = "?" + vulns = [x["id"] for x in r.get("vulns", [])] + bad += bool(vulns) + flag = "VULNERABLE" if vulns else ("newer " if latest not in (v, "?") else "ok ") + print(f" {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}") + return bad + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--debs", default=os.path.join(ROOT, "offline-packages")) + ap.add_argument("--reqs", default=os.path.join(ROOT, "webapp", "requirements.txt")) + ap.add_argument("--all", action="store_true", help="also list non-security updates") + a = ap.parse_args() + try: + problems = check_debs(a.debs, a.all) + check_pins(a.reqs) + except OSError as ex: + print(f"could not check: {ex}", file=sys.stderr) + return 2 + print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/download-packages.sh b/scripts/download-packages.sh index bdd0135..0fa2068 100755 --- a/scripts/download-packages.sh +++ b/scripts/download-packages.sh @@ -19,8 +19,10 @@ OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_D # Detect host Ubuntu codename. Run inside the container if not Noble (24.04). HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")" -if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then - echo "Host is '$HOST_CODENAME', not 'noble' (Ubuntu 24.04)." +# USE_DOCKER=1 forces the container even on a noble host: no sudo needed, +# and the host's own apt state is left alone. +if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then + echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container." echo "Re-running inside ubuntu:24.04 docker container..." echo "" @@ -32,15 +34,21 @@ if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then SCRIPT_PATH="$(readlink -f "$0")" REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)" mkdir -p "$OUT_DIR_ABS" + # Wheels go to their own mount so PIP_DIR is honoured inside the container + # (the in-container default would be the repo's pip-wheels/). + PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)" docker run --rm -i \ -v "$REPO_DIR:/repo" \ -v "$OUT_DIR_ABS:/out" \ + -v "$PIP_DIR_HOST:/pipout" \ -e IN_DOCKER=1 \ + -e PIP_DIR=/pipout \ + -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ -e BLANCCO_KVER="${BLANCCO_KVER:-}" \ -w /repo \ ubuntu:24.04 \ - bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out" + bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc" echo "" echo "============================================" @@ -85,7 +93,8 @@ PLAYBOOK_PACKAGES=( linux-generic-hwe-24.04 "linux-image-${BLANCCO_KVER}" "linux-modules-${BLANCCO_KVER}" - "linux-modules-extra-${BLANCCO_KVER}" + # No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi + # etc.) in linux-modules, and modules-extra-7.0.* does not exist. ) # Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.) @@ -114,9 +123,13 @@ sudo apt-get update -qq # Simulate install to find all dependencies echo "[2/4] Resolving dependencies..." -EXPLICIT_DEPS=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1 \ - | grep "^Inst " \ - | awk '{print $2}') +SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || { + # e.g. a package name that does not exist in noble: say which. + echo "ERROR: apt could not resolve the package list:" >&2 + echo "$SIM" | grep -E "^(E|W):" >&2 + exit 1 +} +EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}') # ALSO pull every package that would upgrade in a dist-upgrade. This is # critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit @@ -147,9 +160,15 @@ echo " $DEB_COUNT packages ($TOTAL_SIZE)" echo "[4/4] Downloading Python wheels for webapp..." # Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" -PIP_DIR="$REPO_ROOT/pip-wheels" -mkdir -p "$PIP_DIR" -pip3 download -d "$PIP_DIR" flask lxml 2>&1 | tail -5 +PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}" +# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's +# Python 3.12 on x86_64. The folder is emptied first: the playbook installs +# every wheel in it and refuses any wheel that is not pinned. +find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR" +pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \ + --only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \ + --platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \ + 2>&1 | tail -5 WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l) echo " $WHL_COUNT Python packages downloaded to pip-wheels/" diff --git a/webapp/requirements.txt b/webapp/requirements.txt index 1e90dc9..0d64502 100644 --- a/webapp/requirements.txt +++ b/webapp/requirements.txt @@ -1,2 +1,24 @@ -flask -lxml +# Webapp Python dependencies, pinned with sha256 for the PXE server's +# Python 3.12 (Ubuntu 24.04, x86_64). scripts/download-packages.sh fetches +# exactly these into pip-wheels/; the playbook refuses any other wheel. +# To update: change a version here, regenerate the hashes +# (pip hash, or the PyPI JSON API), re-run download-packages.sh, and check +# for known vulnerabilities (osv.dev) before building a USB. + +blinker==1.9.0 \ + --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 +flask==3.1.3 \ + --hash=sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c +itsdangerous==2.2.0 \ + --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef +jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +lxml==6.1.3 \ + --hash=sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11 \ + --hash=sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c +markupsafe==3.0.4 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 +werkzeug==3.1.9 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab