ge-aerospace/pxe-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'pxe-webapp-redesign' into usb-build

Browse Source
This commit is contained in:
cproudlock
2026-10-15 09:57:15 -04:00
parent 86b4c1f301 18f20b00e2
commit c1a5fb3e02
4 changed files with 225 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
playbook/pxe_server_setup.yml
View File

@@ -1057,7 +1057,7 @@
files (vmlinuz-ubuntu-{{ blancco_ksuffix }},
kexec-initrd-{{ blancco_ksuffix }}.img,
kmod-{{ blancco_kver }}.tar.gz) are NOT built. Install
linux-image-{{ blancco_kver }} + linux-modules-extra-{{ blancco_kver }}
linux-image-{{ blancco_kver }} + linux-modules-{{ blancco_kver }}
(scripts/download-packages.sh, BLANCCO_KVER) or set blancco_kver
to an installed version (ls /lib/modules), then re-run.
when: not blancco_kernel_ok
@@ -1445,10 +1445,15 @@
mode: '0755'
- name: "Copy webapp from USB"
shell: >
cp -r "{{ usb_root }}/webapp/"* /opt/pxe-webapp/ 2>/dev/null || true
# The whole tree (app, config, services, views, templates, static), every
# run, so a re-run with a newer USB updates the app. Was: cp with
# creates: app.py, which never updated an existing install.
shell: |
set -e
test -f "{{ usb_root }}/webapp/app.py"
rsync -a --delete --exclude __pycache__ "{{ usb_root }}/webapp/" /opt/pxe-webapp/
args:
creates: /opt/pxe-webapp/app.py
executable: /bin/bash
- name: "Install webapp Python dependencies (offline wheels)"
args:
@@ -1465,16 +1470,51 @@
if [ -n "$WHEEL_DIR" ]; then
# Install wheels directly using Python zipfile (bypasses pip entirely)
# This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12
export REQS="{{ usb_root }}/webapp/requirements.txt"
python3 << 'PYEOF'
import zipfile, sysconfig, glob, os
# Installs only the pinned wheels: the sha256 of every wheel must be in
# webapp/requirements.txt, and each pinned package must have exactly one
# wheel. An older installed version of a package is removed first (by its
# RECORD), so an upgrade does not leave the old files beside the new.
import zipfile, sysconfig, glob, os, re, sys, hashlib, csv, shutil
site = sysconfig.get_path('platlib')
wheel_dir = os.environ['WHEEL_DIR']
for whl in sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))):
name = os.path.basename(whl)
print(f' Installing {name}')
reqs = open(os.environ['REQS']).read()
pins, hashes = {}, set()
for m in re.finditer(r'^([A-Za-z0-9_.-]+)==([^\s\\]+)((?:\s*\\?\s*--hash=sha256:[0-9a-f]{64})+)', reqs, re.M):
name = re.sub(r'[-_.]+', '_', m.group(1)).lower()
pins[name] = m.group(2)
hashes.update(re.findall(r'sha256:([0-9a-f]{64})', m.group(3)))
wheels = sorted(glob.glob(os.path.join(wheel_dir, '*.whl')))
seen = {}
for whl in wheels:
h = hashlib.sha256(open(whl, 'rb').read()).hexdigest()
name = re.sub(r'[-_.]+', '_', os.path.basename(whl).split('-')[0]).lower()
if h not in hashes or name not in pins:
sys.exit(f'REFUSED: {os.path.basename(whl)} is not a pinned wheel (sha256 {h[:16]}...)')
seen.setdefault(name, []).append(whl)
bad = [n for n in pins if len(seen.get(n, [])) != 1]
if bad:
sys.exit(f'REFUSED: need exactly one wheel for each pinned package; check: {", ".join(bad)}')
for name, (whl,) in sorted(seen.items()):
for old in glob.glob(os.path.join(site, '*.dist-info')):
stem = os.path.basename(old)[:-len('.dist-info')]
oname = re.sub(r'[-_.]+', '_', stem.rsplit('-', 1)[0]).lower()
if oname != name:
continue
rec = os.path.join(old, 'RECORD')
if os.path.isfile(rec):
for row in csv.reader(open(rec)):
if not row:
continue
f = os.path.normpath(os.path.join(site, row[0]))
if f.startswith(site + os.sep) and os.path.isfile(f):
os.remove(f)
shutil.rmtree(old, ignore_errors=True)
print(f' Installing {os.path.basename(whl)}')
with zipfile.ZipFile(whl) as z:
z.extractall(site)
print('All wheels installed to ' + site)
print('All pinned wheels installed to ' + site)
PYEOF
else
# Fallback: try system pip (works if system has internet and compatible pip)

123
scripts/check-updates.py Executable file
View File

@@ -0,0 +1,123 @@
#!/usr/bin/env python3
"""check-updates.py - is the offline bundle current? Run before building a USB.
* offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates,
noble-security; main + universe): which have a newer version, and which
of those are security updates.
* webapp/requirements.txt pins against PyPI (newer release?) and osv.dev
(known vulnerabilities in the pinned version).
Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found
(the details are printed); 2 = could not check (no network, bad input).
Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only.
Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all]
"""
import argparse
import glob
import gzip
import json
import os
import re
import subprocess
import sys
import urllib.request
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists"
SUITES = ("noble", "noble-updates", "noble-security")
COMPONENTS = ("main", "universe")
def fetch(url, timeout=60):
with urllib.request.urlopen(url, timeout=timeout) as r:
return r.read()
def newer(a, b):
"""True if Debian version a > b."""
return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0
def ubuntu_index():
"""({package: newest version}, {package: newest security version})."""
best, sec = {}, {}
for suite in SUITES:
for comp in COMPONENTS:
text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace")
pkg = None
for line in text.splitlines():
if line.startswith("Package: "):
pkg = line[9:].strip()
elif line.startswith("Version: ") and pkg:
v = line[9:].strip()
if pkg not in best or newer(v, best[pkg]):
best[pkg] = v
if suite.endswith("security") and (pkg not in sec or newer(v, sec[pkg])):
sec[pkg] = v
return best, sec
def check_debs(deb_dir, show_all):
debs = glob.glob(os.path.join(deb_dir, "*.deb"))
if not debs:
print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)")
return 1
best, sec = ubuntu_index()
stale, security = [], []
for d in debs:
parts = os.path.basename(d)[:-4].split("_")
name, ver = parts[0], parts[1].replace("%3a", ":")
if name in best and newer(best[name], ver):
is_sec = name in sec and newer(sec[name], ver)
(security if is_sec else stale).append((name, ver, best[name]))
print(f"Ubuntu packages: {len(debs)} in {deb_dir}; {len(security) + len(stale)} have newer versions, "
f"{len(security)} of them security updates")
for n, v, nv in sorted(security):
print(f" SECURITY {n:34} {v:30} -> {nv}")
if show_all:
for n, v, nv in sorted(stale):
print(f" update {n:34} {v:30} -> {nv}")
return len(security)
def check_pins(reqs):
pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M)
if not pins:
print(f"no pinned packages in {reqs}")
return 0
q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]}
req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(),
headers={"Content-Type": "application/json"})
results = json.load(urllib.request.urlopen(req, timeout=60))["results"]
bad = 0
print(f"Python pins in {os.path.relpath(reqs, ROOT)}:")
for (n, v), r in zip(pins, results):
try:
latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"]
except Exception: # report, keep going
latest = "?"
vulns = [x["id"] for x in r.get("vulns", [])]
bad += bool(vulns)
flag = "VULNERABLE" if vulns else ("newer " if latest not in (v, "?") else "ok ")
print(f" {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}")
return bad
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--debs", default=os.path.join(ROOT, "offline-packages"))
ap.add_argument("--reqs", default=os.path.join(ROOT, "webapp", "requirements.txt"))
ap.add_argument("--all", action="store_true", help="also list non-security updates")
a = ap.parse_args()
try:
problems = check_debs(a.debs, a.all) + check_pins(a.reqs)
except OSError as ex:
print(f"could not check: {ex}", file=sys.stderr)
return 2
print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating")
return 1 if problems else 0
if __name__ == "__main__":
sys.exit(main())

39
scripts/download-packages.sh
View File

@@ -19,8 +19,10 @@ OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_D
# Detect host Ubuntu codename. Run inside the container if not Noble (24.04).
HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")"
if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
echo "Host is '$HOST_CODENAME', not 'noble' (Ubuntu 24.04)."
# USE_DOCKER=1 forces the container even on a noble host: no sudo needed,
# and the host's own apt state is left alone.
if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then
echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container."
echo "Re-running inside ubuntu:24.04 docker container..."
echo ""
@@ -32,15 +34,21 @@ if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
SCRIPT_PATH="$(readlink -f "$0")"
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
mkdir -p "$OUT_DIR_ABS"
# Wheels go to their own mount so PIP_DIR is honoured inside the container
# (the in-container default would be the repo's pip-wheels/).
PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)"
docker run --rm -i \
-v "$REPO_DIR:/repo" \
-v "$OUT_DIR_ABS:/out" \
-v "$PIP_DIR_HOST:/pipout" \
-e IN_DOCKER=1 \
-e PIP_DIR=/pipout \
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
-e BLANCCO_KVER="${BLANCCO_KVER:-}" \
-w /repo \
ubuntu:24.04 \
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out"
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc"
echo ""
echo "============================================"
@@ -85,7 +93,8 @@ PLAYBOOK_PACKAGES=(
linux-generic-hwe-24.04
"linux-image-${BLANCCO_KVER}"
"linux-modules-${BLANCCO_KVER}"
"linux-modules-extra-${BLANCCO_KVER}"
# No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi
# etc.) in linux-modules, and modules-extra-7.0.* does not exist.
)
# Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.)
@@ -114,9 +123,13 @@ sudo apt-get update -qq
# Simulate install to find all dependencies
echo "[2/4] Resolving dependencies..."
EXPLICIT_DEPS=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1 \
| grep "^Inst " \
| awk '{print $2}')
SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || {
# e.g. a package name that does not exist in noble: say which.
echo "ERROR: apt could not resolve the package list:" >&2
echo "$SIM" | grep -E "^(E|W):" >&2
exit 1
}
EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}')
# ALSO pull every package that would upgrade in a dist-upgrade. This is
# critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit
@@ -147,9 +160,15 @@ echo " $DEB_COUNT packages ($TOTAL_SIZE)"
echo "[4/4] Downloading Python wheels for webapp..."
# Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
PIP_DIR="$REPO_ROOT/pip-wheels"
mkdir -p "$PIP_DIR"
pip3 download -d "$PIP_DIR" flask lxml 2>&1 | tail -5
PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}"
# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
# every wheel in it and refuses any wheel that is not pinned.
find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR"
pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \
--only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
--platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \
2>&1 | tail -5
WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l)
echo " $WHL_COUNT Python packages downloaded to pip-wheels/"

26
webapp/requirements.txt
View File

@@ -1,2 +1,24 @@
flask
lxml
# Webapp Python dependencies, pinned with sha256 for the PXE server's
# Python 3.12 (Ubuntu 24.04, x86_64). scripts/download-packages.sh fetches
# exactly these into pip-wheels/; the playbook refuses any other wheel.
# To update: change a version here, regenerate the hashes
# (pip hash, or the PyPI JSON API), re-run download-packages.sh, and check
# for known vulnerabilities (osv.dev) before building a USB.
blinker==1.9.0 \
--hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc
click==8.5.0 \
--hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360
flask==3.1.3 \
--hash=sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c
itsdangerous==2.2.0 \
--hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef
jinja2==3.1.6 \
--hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
lxml==6.1.3 \
--hash=sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11 \
--hash=sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c
markupsafe==3.0.4 \
--hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581
werkzeug==3.1.9 \
--hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab