diff --git a/playbook/pxe_server_setup.yml b/playbook/pxe_server_setup.yml index e09f40b..57724a8 100644 --- a/playbook/pxe_server_setup.yml +++ b/playbook/pxe_server_setup.yml @@ -1057,7 +1057,7 @@ files (vmlinuz-ubuntu-{{ blancco_ksuffix }}, kexec-initrd-{{ blancco_ksuffix }}.img, kmod-{{ blancco_kver }}.tar.gz) are NOT built. Install - linux-image-{{ blancco_kver }} + linux-modules-extra-{{ blancco_kver }} + linux-image-{{ blancco_kver }} + linux-modules-{{ blancco_kver }} (scripts/download-packages.sh, BLANCCO_KVER) or set blancco_kver to an installed version (ls /lib/modules), then re-run. when: not blancco_kernel_ok @@ -1445,10 +1445,15 @@ mode: '0755' - name: "Copy webapp from USB" - shell: > - cp -r "{{ usb_root }}/webapp/"* /opt/pxe-webapp/ 2>/dev/null || true + # The whole tree (app, config, services, views, templates, static), every + # run, so a re-run with a newer USB updates the app. Was: cp with + # creates: app.py, which never updated an existing install. + shell: | + set -e + test -f "{{ usb_root }}/webapp/app.py" + rsync -a --delete --exclude __pycache__ "{{ usb_root }}/webapp/" /opt/pxe-webapp/ args: - creates: /opt/pxe-webapp/app.py + executable: /bin/bash - name: "Install webapp Python dependencies (offline wheels)" args: @@ -1465,16 +1470,51 @@ if [ -n "$WHEEL_DIR" ]; then # Install wheels directly using Python zipfile (bypasses pip entirely) # This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12 + export REQS="{{ usb_root }}/webapp/requirements.txt" python3 << 'PYEOF' - import zipfile, sysconfig, glob, os + # Installs only the pinned wheels: the sha256 of every wheel must be in + # webapp/requirements.txt, and each pinned package must have exactly one + # wheel. An older installed version of a package is removed first (by its + # RECORD), so an upgrade does not leave the old files beside the new. + import zipfile, sysconfig, glob, os, re, sys, hashlib, csv, shutil site = sysconfig.get_path('platlib') wheel_dir = os.environ['WHEEL_DIR'] - for whl in sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))): - name = os.path.basename(whl) - print(f' Installing {name}') + reqs = open(os.environ['REQS']).read() + pins, hashes = {}, set() + for m in re.finditer(r'^([A-Za-z0-9_.-]+)==([^\s\\]+)((?:\s*\\?\s*--hash=sha256:[0-9a-f]{64})+)', reqs, re.M): + name = re.sub(r'[-_.]+', '_', m.group(1)).lower() + pins[name] = m.group(2) + hashes.update(re.findall(r'sha256:([0-9a-f]{64})', m.group(3))) + wheels = sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))) + seen = {} + for whl in wheels: + h = hashlib.sha256(open(whl, 'rb').read()).hexdigest() + name = re.sub(r'[-_.]+', '_', os.path.basename(whl).split('-')[0]).lower() + if h not in hashes or name not in pins: + sys.exit(f'REFUSED: {os.path.basename(whl)} is not a pinned wheel (sha256 {h[:16]}...)') + seen.setdefault(name, []).append(whl) + bad = [n for n in pins if len(seen.get(n, [])) != 1] + if bad: + sys.exit(f'REFUSED: need exactly one wheel for each pinned package; check: {", ".join(bad)}') + for name, (whl,) in sorted(seen.items()): + for old in glob.glob(os.path.join(site, '*.dist-info')): + stem = os.path.basename(old)[:-len('.dist-info')] + oname = re.sub(r'[-_.]+', '_', stem.rsplit('-', 1)[0]).lower() + if oname != name: + continue + rec = os.path.join(old, 'RECORD') + if os.path.isfile(rec): + for row in csv.reader(open(rec)): + if not row: + continue + f = os.path.normpath(os.path.join(site, row[0])) + if f.startswith(site + os.sep) and os.path.isfile(f): + os.remove(f) + shutil.rmtree(old, ignore_errors=True) + print(f' Installing {os.path.basename(whl)}') with zipfile.ZipFile(whl) as z: z.extractall(site) - print('All wheels installed to ' + site) + print('All pinned wheels installed to ' + site) PYEOF else # Fallback: try system pip (works if system has internet and compatible pip) diff --git a/scripts/check-updates.py b/scripts/check-updates.py new file mode 100755 index 0000000..29f7bf5 --- /dev/null +++ b/scripts/check-updates.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""check-updates.py - is the offline bundle current? Run before building a USB. + + * offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates, + noble-security; main + universe): which have a newer version, and which + of those are security updates. + * webapp/requirements.txt pins against PyPI (newer release?) and osv.dev + (known vulnerabilities in the pinned version). + +Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found +(the details are printed); 2 = could not check (no network, bad input). +Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only. + +Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all] +""" +import argparse +import glob +import gzip +import json +import os +import re +import subprocess +import sys +import urllib.request + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists" +SUITES = ("noble", "noble-updates", "noble-security") +COMPONENTS = ("main", "universe") + + +def fetch(url, timeout=60): + with urllib.request.urlopen(url, timeout=timeout) as r: + return r.read() + + +def newer(a, b): + """True if Debian version a > b.""" + return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0 + + +def ubuntu_index(): + """({package: newest version}, {package: newest security version}).""" + best, sec = {}, {} + for suite in SUITES: + for comp in COMPONENTS: + text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace") + pkg = None + for line in text.splitlines(): + if line.startswith("Package: "): + pkg = line[9:].strip() + elif line.startswith("Version: ") and pkg: + v = line[9:].strip() + if pkg not in best or newer(v, best[pkg]): + best[pkg] = v + if suite.endswith("security") and (pkg not in sec or newer(v, sec[pkg])): + sec[pkg] = v + return best, sec + + +def check_debs(deb_dir, show_all): + debs = glob.glob(os.path.join(deb_dir, "*.deb")) + if not debs: + print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)") + return 1 + best, sec = ubuntu_index() + stale, security = [], [] + for d in debs: + parts = os.path.basename(d)[:-4].split("_") + name, ver = parts[0], parts[1].replace("%3a", ":") + if name in best and newer(best[name], ver): + is_sec = name in sec and newer(sec[name], ver) + (security if is_sec else stale).append((name, ver, best[name])) + print(f"Ubuntu packages: {len(debs)} in {deb_dir}; {len(security) + len(stale)} have newer versions, " + f"{len(security)} of them security updates") + for n, v, nv in sorted(security): + print(f" SECURITY {n:34} {v:30} -> {nv}") + if show_all: + for n, v, nv in sorted(stale): + print(f" update {n:34} {v:30} -> {nv}") + return len(security) + + +def check_pins(reqs): + pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M) + if not pins: + print(f"no pinned packages in {reqs}") + return 0 + q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]} + req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(), + headers={"Content-Type": "application/json"}) + results = json.load(urllib.request.urlopen(req, timeout=60))["results"] + bad = 0 + print(f"Python pins in {os.path.relpath(reqs, ROOT)}:") + for (n, v), r in zip(pins, results): + try: + latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"] + except Exception: # report, keep going + latest = "?" + vulns = [x["id"] for x in r.get("vulns", [])] + bad += bool(vulns) + flag = "VULNERABLE" if vulns else ("newer " if latest not in (v, "?") else "ok ") + print(f" {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}") + return bad + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--debs", default=os.path.join(ROOT, "offline-packages")) + ap.add_argument("--reqs", default=os.path.join(ROOT, "webapp", "requirements.txt")) + ap.add_argument("--all", action="store_true", help="also list non-security updates") + a = ap.parse_args() + try: + problems = check_debs(a.debs, a.all) + check_pins(a.reqs) + except OSError as ex: + print(f"could not check: {ex}", file=sys.stderr) + return 2 + print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/download-packages.sh b/scripts/download-packages.sh index bdd0135..0fa2068 100755 --- a/scripts/download-packages.sh +++ b/scripts/download-packages.sh @@ -19,8 +19,10 @@ OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_D # Detect host Ubuntu codename. Run inside the container if not Noble (24.04). HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")" -if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then - echo "Host is '$HOST_CODENAME', not 'noble' (Ubuntu 24.04)." +# USE_DOCKER=1 forces the container even on a noble host: no sudo needed, +# and the host's own apt state is left alone. +if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then + echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container." echo "Re-running inside ubuntu:24.04 docker container..." echo "" @@ -32,15 +34,21 @@ if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then SCRIPT_PATH="$(readlink -f "$0")" REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)" mkdir -p "$OUT_DIR_ABS" + # Wheels go to their own mount so PIP_DIR is honoured inside the container + # (the in-container default would be the repo's pip-wheels/). + PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)" docker run --rm -i \ -v "$REPO_DIR:/repo" \ -v "$OUT_DIR_ABS:/out" \ + -v "$PIP_DIR_HOST:/pipout" \ -e IN_DOCKER=1 \ + -e PIP_DIR=/pipout \ + -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ -e BLANCCO_KVER="${BLANCCO_KVER:-}" \ -w /repo \ ubuntu:24.04 \ - bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out" + bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc" echo "" echo "============================================" @@ -85,7 +93,8 @@ PLAYBOOK_PACKAGES=( linux-generic-hwe-24.04 "linux-image-${BLANCCO_KVER}" "linux-modules-${BLANCCO_KVER}" - "linux-modules-extra-${BLANCCO_KVER}" + # No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi + # etc.) in linux-modules, and modules-extra-7.0.* does not exist. ) # Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.) @@ -114,9 +123,13 @@ sudo apt-get update -qq # Simulate install to find all dependencies echo "[2/4] Resolving dependencies..." -EXPLICIT_DEPS=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1 \ - | grep "^Inst " \ - | awk '{print $2}') +SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || { + # e.g. a package name that does not exist in noble: say which. + echo "ERROR: apt could not resolve the package list:" >&2 + echo "$SIM" | grep -E "^(E|W):" >&2 + exit 1 +} +EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}') # ALSO pull every package that would upgrade in a dist-upgrade. This is # critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit @@ -147,9 +160,15 @@ echo " $DEB_COUNT packages ($TOTAL_SIZE)" echo "[4/4] Downloading Python wheels for webapp..." # Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" -PIP_DIR="$REPO_ROOT/pip-wheels" -mkdir -p "$PIP_DIR" -pip3 download -d "$PIP_DIR" flask lxml 2>&1 | tail -5 +PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}" +# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's +# Python 3.12 on x86_64. The folder is emptied first: the playbook installs +# every wheel in it and refuses any wheel that is not pinned. +find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR" +pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \ + --only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \ + --platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \ + 2>&1 | tail -5 WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l) echo " $WHL_COUNT Python packages downloaded to pip-wheels/" diff --git a/webapp/requirements.txt b/webapp/requirements.txt index 1e90dc9..0d64502 100644 --- a/webapp/requirements.txt +++ b/webapp/requirements.txt @@ -1,2 +1,24 @@ -flask -lxml +# Webapp Python dependencies, pinned with sha256 for the PXE server's +# Python 3.12 (Ubuntu 24.04, x86_64). scripts/download-packages.sh fetches +# exactly these into pip-wheels/; the playbook refuses any other wheel. +# To update: change a version here, regenerate the hashes +# (pip hash, or the PyPI JSON API), re-run download-packages.sh, and check +# for known vulnerabilities (osv.dev) before building a USB. + +blinker==1.9.0 \ + --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 +flask==3.1.3 \ + --hash=sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c +itsdangerous==2.2.0 \ + --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef +jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +lxml==6.1.3 \ + --hash=sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11 \ + --hash=sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c +markupsafe==3.0.4 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 +werkzeug==3.1.9 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab