Merge branch 'pxe-webapp-redesign' into usb-build

playbook/pxe_server_setup.yml

@@ -1057,7 +1057,7 @@
           files (vmlinuz-ubuntu-{{ blancco_ksuffix }},
           kexec-initrd-{{ blancco_ksuffix }}.img,
           kmod-{{ blancco_kver }}.tar.gz) are NOT built. Install
-          linux-image-{{ blancco_kver }} + linux-modules-extra-{{ blancco_kver }}
+          linux-image-{{ blancco_kver }} + linux-modules-{{ blancco_kver }}
           (scripts/download-packages.sh, BLANCCO_KVER) or set blancco_kver
           to an installed version (ls /lib/modules), then re-run.
       when: not blancco_kernel_ok
@@ -1445,10 +1445,15 @@
         mode: '0755'
 
     - name: "Copy webapp from USB"
-      shell: >
+      # The whole tree (app, config, services, views, templates, static), every
-        cp -r "{{ usb_root }}/webapp/"* /opt/pxe-webapp/ 2>/dev/null || true
+      # run, so a re-run with a newer USB updates the app. Was: cp with
+      # creates: app.py, which never updated an existing install.
+      shell: |
+        set -e
+        test -f "{{ usb_root }}/webapp/app.py"
+        rsync -a --delete --exclude __pycache__ "{{ usb_root }}/webapp/" /opt/pxe-webapp/
       args:
-        creates: /opt/pxe-webapp/app.py
+        executable: /bin/bash
 
     - name: "Install webapp Python dependencies (offline wheels)"
       args:
@@ -1465,16 +1470,51 @@
         if [ -n "$WHEEL_DIR" ]; then
           # Install wheels directly using Python zipfile (bypasses pip entirely)
           # This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12
+          export REQS="{{ usb_root }}/webapp/requirements.txt"
           python3 << 'PYEOF'
-        import zipfile, sysconfig, glob, os
+        # Installs only the pinned wheels: the sha256 of every wheel must be in
+        # webapp/requirements.txt, and each pinned package must have exactly one
+        # wheel. An older installed version of a package is removed first (by its
+        # RECORD), so an upgrade does not leave the old files beside the new.
+        import zipfile, sysconfig, glob, os, re, sys, hashlib, csv, shutil
         site = sysconfig.get_path('platlib')
         wheel_dir = os.environ['WHEEL_DIR']
-        for whl in sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))):
+        reqs = open(os.environ['REQS']).read()
-            name = os.path.basename(whl)
+        pins, hashes = {}, set()
-            print(f'  Installing {name}')
+        for m in re.finditer(r'^([A-Za-z0-9_.-]+)==([^\s\\]+)((?:\s*\\?\s*--hash=sha256:[0-9a-f]{64})+)', reqs, re.M):
+            name = re.sub(r'[-_.]+', '_', m.group(1)).lower()
+            pins[name] = m.group(2)
+            hashes.update(re.findall(r'sha256:([0-9a-f]{64})', m.group(3)))
+        wheels = sorted(glob.glob(os.path.join(wheel_dir, '*.whl')))
+        seen = {}
+        for whl in wheels:
+            h = hashlib.sha256(open(whl, 'rb').read()).hexdigest()
+            name = re.sub(r'[-_.]+', '_', os.path.basename(whl).split('-')[0]).lower()
+            if h not in hashes or name not in pins:
+                sys.exit(f'REFUSED: {os.path.basename(whl)} is not a pinned wheel (sha256 {h[:16]}...)')
+            seen.setdefault(name, []).append(whl)
+        bad = [n for n in pins if len(seen.get(n, [])) != 1]
+        if bad:
+            sys.exit(f'REFUSED: need exactly one wheel for each pinned package; check: {", ".join(bad)}')
+        for name, (whl,) in sorted(seen.items()):
+            for old in glob.glob(os.path.join(site, '*.dist-info')):
+                stem = os.path.basename(old)[:-len('.dist-info')]
+                oname = re.sub(r'[-_.]+', '_', stem.rsplit('-', 1)[0]).lower()
+                if oname != name:
+                    continue
+                rec = os.path.join(old, 'RECORD')
+                if os.path.isfile(rec):
+                    for row in csv.reader(open(rec)):
+                        if not row:
+                            continue
+                        f = os.path.normpath(os.path.join(site, row[0]))
+                        if f.startswith(site + os.sep) and os.path.isfile(f):
+                            os.remove(f)
+                shutil.rmtree(old, ignore_errors=True)
+            print(f'  Installing {os.path.basename(whl)}')
             with zipfile.ZipFile(whl) as z:
                 z.extractall(site)
-        print('All wheels installed to ' + site)
+        print('All pinned wheels installed to ' + site)
         PYEOF
         else
           # Fallback: try system pip (works if system has internet and compatible pip)

scripts/check-updates.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""check-updates.py - is the offline bundle current? Run before building a USB.
+
+  * offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates,
+    noble-security; main + universe): which have a newer version, and which
+    of those are security updates.
+  * webapp/requirements.txt pins against PyPI (newer release?) and osv.dev
+    (known vulnerabilities in the pinned version).
+
+Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found
+(the details are printed); 2 = could not check (no network, bad input).
+Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only.
+
+Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all]
+"""
+import argparse
+import glob
+import gzip
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists"
+SUITES = ("noble", "noble-updates", "noble-security")
+COMPONENTS = ("main", "universe")
+
+
+def fetch(url, timeout=60):
+    with urllib.request.urlopen(url, timeout=timeout) as r:
+        return r.read()
+
+
+def newer(a, b):
+    """True if Debian version a > b."""
+    return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0
+
+
+def ubuntu_index():
+    """({package: newest version}, {package: newest security version})."""
+    best, sec = {}, {}
+    for suite in SUITES:
+        for comp in COMPONENTS:
+            text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace")
+            pkg = None
+            for line in text.splitlines():
+                if line.startswith("Package: "):
+                    pkg = line[9:].strip()
+                elif line.startswith("Version: ") and pkg:
+                    v = line[9:].strip()
+                    if pkg not in best or newer(v, best[pkg]):
+                        best[pkg] = v
+                    if suite.endswith("security") and (pkg not in sec or newer(v, sec[pkg])):
+                        sec[pkg] = v
+    return best, sec
+
+
+def check_debs(deb_dir, show_all):
+    debs = glob.glob(os.path.join(deb_dir, "*.deb"))
+    if not debs:
+        print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)")
+        return 1
+    best, sec = ubuntu_index()
+    stale, security = [], []
+    for d in debs:
+        parts = os.path.basename(d)[:-4].split("_")
+        name, ver = parts[0], parts[1].replace("%3a", ":")
+        if name in best and newer(best[name], ver):
+            is_sec = name in sec and newer(sec[name], ver)
+            (security if is_sec else stale).append((name, ver, best[name]))
+    print(f"Ubuntu packages: {len(debs)} in {deb_dir}; {len(security) + len(stale)} have newer versions, "
+          f"{len(security)} of them security updates")
+    for n, v, nv in sorted(security):
+        print(f"  SECURITY {n:34} {v:30} -> {nv}")
+    if show_all:
+        for n, v, nv in sorted(stale):
+            print(f"  update   {n:34} {v:30} -> {nv}")
+    return len(security)
+
+
+def check_pins(reqs):
+    pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M)
+    if not pins:
+        print(f"no pinned packages in {reqs}")
+        return 0
+    q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]}
+    req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(),
+                                 headers={"Content-Type": "application/json"})
+    results = json.load(urllib.request.urlopen(req, timeout=60))["results"]
+    bad = 0
+    print(f"Python pins in {os.path.relpath(reqs, ROOT)}:")
+    for (n, v), r in zip(pins, results):
+        try:
+            latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"]
+        except Exception:  # report, keep going
+            latest = "?"
+        vulns = [x["id"] for x in r.get("vulns", [])]
+        bad += bool(vulns)
+        flag = "VULNERABLE" if vulns else ("newer     " if latest not in (v, "?") else "ok        ")
+        print(f"  {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}")
+    return bad
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("--debs", default=os.path.join(ROOT, "offline-packages"))
+    ap.add_argument("--reqs", default=os.path.join(ROOT, "webapp", "requirements.txt"))
+    ap.add_argument("--all", action="store_true", help="also list non-security updates")
+    a = ap.parse_args()
+    try:
+        problems = check_debs(a.debs, a.all) + check_pins(a.reqs)
+    except OSError as ex:
+        print(f"could not check: {ex}", file=sys.stderr)
+        return 2
+    print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating")
+    return 1 if problems else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/download-packages.sh

@@ -19,8 +19,10 @@ OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_D
 # Detect host Ubuntu codename. Run inside the container if not Noble (24.04).
 HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")"
 
-if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
+# USE_DOCKER=1 forces the container even on a noble host: no sudo needed,
-    echo "Host is '$HOST_CODENAME', not 'noble' (Ubuntu 24.04)."
+# and the host's own apt state is left alone.
+if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then
+    echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container."
     echo "Re-running inside ubuntu:24.04 docker container..."
     echo ""
 
@@ -32,15 +34,21 @@ if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
     SCRIPT_PATH="$(readlink -f "$0")"
     REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
     mkdir -p "$OUT_DIR_ABS"
+    # Wheels go to their own mount so PIP_DIR is honoured inside the container
+    # (the in-container default would be the repo's pip-wheels/).
+    PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)"
 
     docker run --rm -i \
         -v "$REPO_DIR:/repo" \
         -v "$OUT_DIR_ABS:/out" \
+        -v "$PIP_DIR_HOST:/pipout" \
         -e IN_DOCKER=1 \
+        -e PIP_DIR=/pipout \
+        -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
         -e BLANCCO_KVER="${BLANCCO_KVER:-}" \
         -w /repo \
         ubuntu:24.04 \
-        bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out"
+        bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc"
 
     echo ""
     echo "============================================"
@@ -85,7 +93,8 @@ PLAYBOOK_PACKAGES=(
     linux-generic-hwe-24.04
     "linux-image-${BLANCCO_KVER}"
     "linux-modules-${BLANCCO_KVER}"
-    "linux-modules-extra-${BLANCCO_KVER}"
+    # No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi
+    # etc.) in linux-modules, and modules-extra-7.0.* does not exist.
 )
 
 # Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.)
@@ -114,9 +123,13 @@ sudo apt-get update -qq
 
 # Simulate install to find all dependencies
 echo "[2/4] Resolving dependencies..."
-EXPLICIT_DEPS=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1 \
+SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || {
-    | grep "^Inst " \
+    # e.g. a package name that does not exist in noble: say which.
-    | awk '{print $2}')
+    echo "ERROR: apt could not resolve the package list:" >&2
+    echo "$SIM" | grep -E "^(E|W):" >&2
+    exit 1
+}
+EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}')
 
 # ALSO pull every package that would upgrade in a dist-upgrade. This is
 # critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit
@@ -147,9 +160,15 @@ echo "    $DEB_COUNT packages ($TOTAL_SIZE)"
 echo "[4/4] Downloading Python wheels for webapp..."
 # Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR
 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-PIP_DIR="$REPO_ROOT/pip-wheels"
+PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}"
-mkdir -p "$PIP_DIR"
+# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
-pip3 download -d "$PIP_DIR" flask lxml 2>&1 | tail -5
+# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
+# every wheel in it and refuses any wheel that is not pinned.
+find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR"
+pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \
+    --only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
+    --platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \
+    2>&1 | tail -5
 
 WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l)
 echo "    $WHL_COUNT Python packages downloaded to pip-wheels/"

webapp/requirements.txt

@@ -1,2 +1,24 @@
-flask
+# Webapp Python dependencies, pinned with sha256 for the PXE server's
-lxml
+# Python 3.12 (Ubuntu 24.04, x86_64). scripts/download-packages.sh fetches
+# exactly these into pip-wheels/; the playbook refuses any other wheel.
+# To update: change a version here, regenerate the hashes
+# (pip hash, or the PyPI JSON API), re-run download-packages.sh, and check
+# for known vulnerabilities (osv.dev) before building a USB.
+
+blinker==1.9.0 \
+    --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc
+click==8.5.0 \
+    --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360
+flask==3.1.3 \
+    --hash=sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c
+itsdangerous==2.2.0 \
+    --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef
+jinja2==3.1.6 \
+    --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
+lxml==6.1.3 \
+    --hash=sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11 \
+    --hash=sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c
+markupsafe==3.0.4 \
+    --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581
+werkzeug==3.1.9 \
+    --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab