Offline bundle: pin and hash the webapp wheels, refuse unpinned ones, check for updates
- webapp/requirements.txt pins all eight wheels with sha256, at the fixed releases (lxml 6.1.3 for GHSA-vfmq-68hx-4jfw / PYSEC-2026-87, werkzeug 3.1.9 for GHSA-g6x2-hccm-hh4m). - download-packages.sh fetches exactly that set for Python 3.12 / x86_64 into an emptied folder; PIP_DIR is honoured inside the container (own mount), files are handed back to the calling user, USE_DOCKER=1 forces the container on a noble host, an unknown package name now prints apt's error instead of failing silently, and linux-modules-extra-7.0.* is gone (it does not exist; 7.0 ships those modules in linux-modules). - Playbook: the wheel installer refuses any wheel whose sha256 is not pinned, needs exactly one wheel per pin, and removes an older installed version by its RECORD first. The webapp copy is a full rsync every run (was cp with creates: app.py, which never updated an install). - scripts/check-updates.py: offline debs against noble security/updates and the pins against PyPI + osv.dev; exit 1 when something needs updating. Run it before building a USB. Tested: installer in a Python 3.12 venv (clean install, upgrade over lxml 6.0.4 + werkzeug 3.1.8 leaves no old dist-info, re-run is a no-op, a stray wheel is refused); check-updates.py flags the April bundle (94 security updates) and passes the pins.
This commit is contained in:
cproudlock
123
scripts/check-updates.py
Executable file
123
scripts/check-updates.py
Executable file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""check-updates.py - is the offline bundle current? Run before building a USB.
|
||||
|
||||
* offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates,
|
||||
noble-security; main + universe): which have a newer version, and which
|
||||
of those are security updates.
|
||||
* webapp/requirements.txt pins against PyPI (newer release?) and osv.dev
|
||||
(known vulnerabilities in the pinned version).
|
||||
|
||||
Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found
|
||||
(the details are printed); 2 = could not check (no network, bad input).
|
||||
Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only.
|
||||
|
||||
Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all]
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists"
|
||||
SUITES = ("noble", "noble-updates", "noble-security")
|
||||
COMPONENTS = ("main", "universe")
|
||||
|
||||
|
||||
def fetch(url, timeout=60):
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def newer(a, b):
|
||||
"""True if Debian version a > b."""
|
||||
return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0
|
||||
|
||||
|
||||
def ubuntu_index():
|
||||
"""({package: newest version}, {package: newest security version})."""
|
||||
best, sec = {}, {}
|
||||
for suite in SUITES:
|
||||
for comp in COMPONENTS:
|
||||
text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace")
|
||||
pkg = None
|
||||
for line in text.splitlines():
|
||||
if line.startswith("Package: "):
|
||||
pkg = line[9:].strip()
|
||||
elif line.startswith("Version: ") and pkg:
|
||||
v = line[9:].strip()
|
||||
if pkg not in best or newer(v, best[pkg]):
|
||||
best[pkg] = v
|
||||
if suite.endswith("security") and (pkg not in sec or newer(v, sec[pkg])):
|
||||
sec[pkg] = v
|
||||
return best, sec
|
||||
|
||||
|
||||
def check_debs(deb_dir, show_all):
|
||||
debs = glob.glob(os.path.join(deb_dir, "*.deb"))
|
||||
if not debs:
|
||||
print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)")
|
||||
return 1
|
||||
best, sec = ubuntu_index()
|
||||
stale, security = [], []
|
||||
for d in debs:
|
||||
parts = os.path.basename(d)[:-4].split("_")
|
||||
name, ver = parts[0], parts[1].replace("%3a", ":")
|
||||
if name in best and newer(best[name], ver):
|
||||
is_sec = name in sec and newer(sec[name], ver)
|
||||
(security if is_sec else stale).append((name, ver, best[name]))
|
||||
print(f"Ubuntu packages: {len(debs)} in {deb_dir}; {len(security) + len(stale)} have newer versions, "
|
||||
f"{len(security)} of them security updates")
|
||||
for n, v, nv in sorted(security):
|
||||
print(f" SECURITY {n:34} {v:30} -> {nv}")
|
||||
if show_all:
|
||||
for n, v, nv in sorted(stale):
|
||||
print(f" update {n:34} {v:30} -> {nv}")
|
||||
return len(security)
|
||||
|
||||
|
||||
def check_pins(reqs):
|
||||
pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M)
|
||||
if not pins:
|
||||
print(f"no pinned packages in {reqs}")
|
||||
return 0
|
||||
q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]}
|
||||
req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
results = json.load(urllib.request.urlopen(req, timeout=60))["results"]
|
||||
bad = 0
|
||||
print(f"Python pins in {os.path.relpath(reqs, ROOT)}:")
|
||||
for (n, v), r in zip(pins, results):
|
||||
try:
|
||||
latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"]
|
||||
except Exception: # report, keep going
|
||||
latest = "?"
|
||||
vulns = [x["id"] for x in r.get("vulns", [])]
|
||||
bad += bool(vulns)
|
||||
flag = "VULNERABLE" if vulns else ("newer " if latest not in (v, "?") else "ok ")
|
||||
print(f" {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}")
|
||||
return bad
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--debs", default=os.path.join(ROOT, "offline-packages"))
|
||||
ap.add_argument("--reqs", default=os.path.join(ROOT, "webapp", "requirements.txt"))
|
||||
ap.add_argument("--all", action="store_true", help="also list non-security updates")
|
||||
a = ap.parse_args()
|
||||
try:
|
||||
problems = check_debs(a.debs, a.all) + check_pins(a.reqs)
|
||||
except OSError as ex:
|
||||
print(f"could not check: {ex}", file=sys.stderr)
|
||||
return 2
|
||||
print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -19,8 +19,10 @@ OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_D
|
||||
# Detect host Ubuntu codename. Run inside the container if not Noble (24.04).
|
||||
HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")"
|
||||
|
||||
if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
|
||||
echo "Host is '$HOST_CODENAME', not 'noble' (Ubuntu 24.04)."
|
||||
# USE_DOCKER=1 forces the container even on a noble host: no sudo needed,
|
||||
# and the host's own apt state is left alone.
|
||||
if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then
|
||||
echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container."
|
||||
echo "Re-running inside ubuntu:24.04 docker container..."
|
||||
echo ""
|
||||
|
||||
@@ -32,15 +34,21 @@ if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
|
||||
SCRIPT_PATH="$(readlink -f "$0")"
|
||||
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
|
||||
mkdir -p "$OUT_DIR_ABS"
|
||||
# Wheels go to their own mount so PIP_DIR is honoured inside the container
|
||||
# (the in-container default would be the repo's pip-wheels/).
|
||||
PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)"
|
||||
|
||||
docker run --rm -i \
|
||||
-v "$REPO_DIR:/repo" \
|
||||
-v "$OUT_DIR_ABS:/out" \
|
||||
-v "$PIP_DIR_HOST:/pipout" \
|
||||
-e IN_DOCKER=1 \
|
||||
-e PIP_DIR=/pipout \
|
||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
||||
-e BLANCCO_KVER="${BLANCCO_KVER:-}" \
|
||||
-w /repo \
|
||||
ubuntu:24.04 \
|
||||
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out"
|
||||
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc"
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
@@ -85,7 +93,8 @@ PLAYBOOK_PACKAGES=(
|
||||
linux-generic-hwe-24.04
|
||||
"linux-image-${BLANCCO_KVER}"
|
||||
"linux-modules-${BLANCCO_KVER}"
|
||||
"linux-modules-extra-${BLANCCO_KVER}"
|
||||
# No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi
|
||||
# etc.) in linux-modules, and modules-extra-7.0.* does not exist.
|
||||
)
|
||||
|
||||
# Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.)
|
||||
@@ -114,9 +123,13 @@ sudo apt-get update -qq
|
||||
|
||||
# Simulate install to find all dependencies
|
||||
echo "[2/4] Resolving dependencies..."
|
||||
EXPLICIT_DEPS=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1 \
|
||||
| grep "^Inst " \
|
||||
| awk '{print $2}')
|
||||
SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || {
|
||||
# e.g. a package name that does not exist in noble: say which.
|
||||
echo "ERROR: apt could not resolve the package list:" >&2
|
||||
echo "$SIM" | grep -E "^(E|W):" >&2
|
||||
exit 1
|
||||
}
|
||||
EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}')
|
||||
|
||||
# ALSO pull every package that would upgrade in a dist-upgrade. This is
|
||||
# critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit
|
||||
@@ -147,9 +160,15 @@ echo " $DEB_COUNT packages ($TOTAL_SIZE)"
|
||||
echo "[4/4] Downloading Python wheels for webapp..."
|
||||
# Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
PIP_DIR="$REPO_ROOT/pip-wheels"
|
||||
mkdir -p "$PIP_DIR"
|
||||
pip3 download -d "$PIP_DIR" flask lxml 2>&1 | tail -5
|
||||
PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}"
|
||||
# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
|
||||
# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
|
||||
# every wheel in it and refuses any wheel that is not pinned.
|
||||
find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR"
|
||||
pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \
|
||||
--only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
|
||||
--platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \
|
||||
2>&1 | tail -5
|
||||
|
||||
WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l)
|
||||
echo " $WHL_COUNT Python packages downloaded to pip-wheels/"
|
||||
|
||||
Reference in New Issue
Block a user