Merge branch 'pxe-webapp-redesign' into usb-build
This commit is contained in:
cproudlock
123
scripts/check-updates.py
Executable file
123
scripts/check-updates.py
Executable file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""check-updates.py - is the offline bundle current? Run before building a USB.
|
||||
|
||||
* offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates,
|
||||
noble-security; main + universe): which have a newer version, and which
|
||||
of those are security updates.
|
||||
* webapp/requirements.txt pins against PyPI (newer release?) and osv.dev
|
||||
(known vulnerabilities in the pinned version).
|
||||
|
||||
Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found
|
||||
(the details are printed); 2 = could not check (no network, bad input).
|
||||
Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only.
|
||||
|
||||
Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all]
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists"
|
||||
SUITES = ("noble", "noble-updates", "noble-security")
|
||||
COMPONENTS = ("main", "universe")
|
||||
|
||||
|
||||
def fetch(url, timeout=60):
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def newer(a, b):
|
||||
"""True if Debian version a > b."""
|
||||
return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0
|
||||
|
||||
|
||||
def ubuntu_index():
|
||||
"""({package: newest version}, {package: newest security version})."""
|
||||
best, sec = {}, {}
|
||||
for suite in SUITES:
|
||||
for comp in COMPONENTS:
|
||||
text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace")
|
||||
pkg = None
|
||||
for line in text.splitlines():
|
||||
if line.startswith("Package: "):
|
||||
pkg = line[9:].strip()
|
||||
elif line.startswith("Version: ") and pkg:
|
||||
v = line[9:].strip()
|
||||
if pkg not in best or newer(v, best[pkg]):
|
||||
best[pkg] = v
|
||||
if suite.endswith("security") and (pkg not in sec or newer(v, sec[pkg])):
|
||||
sec[pkg] = v
|
||||
return best, sec
|
||||
|
||||
|
||||
def check_debs(deb_dir, show_all):
|
||||
debs = glob.glob(os.path.join(deb_dir, "*.deb"))
|
||||
if not debs:
|
||||
print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)")
|
||||
return 1
|
||||
best, sec = ubuntu_index()
|
||||
stale, security = [], []
|
||||
for d in debs:
|
||||
parts = os.path.basename(d)[:-4].split("_")
|
||||
name, ver = parts[0], parts[1].replace("%3a", ":")
|
||||
if name in best and newer(best[name], ver):
|
||||
is_sec = name in sec and newer(sec[name], ver)
|
||||
(security if is_sec else stale).append((name, ver, best[name]))
|
||||
print(f"Ubuntu packages: {len(debs)} in {deb_dir}; {len(security) + len(stale)} have newer versions, "
|
||||
f"{len(security)} of them security updates")
|
||||
for n, v, nv in sorted(security):
|
||||
print(f" SECURITY {n:34} {v:30} -> {nv}")
|
||||
if show_all:
|
||||
for n, v, nv in sorted(stale):
|
||||
print(f" update {n:34} {v:30} -> {nv}")
|
||||
return len(security)
|
||||
|
||||
|
||||
def check_pins(reqs):
|
||||
pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M)
|
||||
if not pins:
|
||||
print(f"no pinned packages in {reqs}")
|
||||
return 0
|
||||
q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]}
|
||||
req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
results = json.load(urllib.request.urlopen(req, timeout=60))["results"]
|
||||
bad = 0
|
||||
print(f"Python pins in {os.path.relpath(reqs, ROOT)}:")
|
||||
for (n, v), r in zip(pins, results):
|
||||
try:
|
||||
latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"]
|
||||
except Exception: # report, keep going
|
||||
latest = "?"
|
||||
vulns = [x["id"] for x in r.get("vulns", [])]
|
||||
bad += bool(vulns)
|
||||
flag = "VULNERABLE" if vulns else ("newer " if latest not in (v, "?") else "ok ")
|
||||
print(f" {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}")
|
||||
return bad
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--debs", default=os.path.join(ROOT, "offline-packages"))
|
||||
ap.add_argument("--reqs", default=os.path.join(ROOT, "webapp", "requirements.txt"))
|
||||
ap.add_argument("--all", action="store_true", help="also list non-security updates")
|
||||
a = ap.parse_args()
|
||||
try:
|
||||
problems = check_debs(a.debs, a.all) + check_pins(a.reqs)
|
||||
except OSError as ex:
|
||||
print(f"could not check: {ex}", file=sys.stderr)
|
||||
return 2
|
||||
print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -19,8 +19,10 @@ OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_D
|
||||
# Detect host Ubuntu codename. Run inside the container if not Noble (24.04).
|
||||
HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")"
|
||||
|
||||
if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
|
||||
echo "Host is '$HOST_CODENAME', not 'noble' (Ubuntu 24.04)."
|
||||
# USE_DOCKER=1 forces the container even on a noble host: no sudo needed,
|
||||
# and the host's own apt state is left alone.
|
||||
if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then
|
||||
echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container."
|
||||
echo "Re-running inside ubuntu:24.04 docker container..."
|
||||
echo ""
|
||||
|
||||
@@ -32,15 +34,21 @@ if [ "${IN_DOCKER:-}" != "1" ] && [ "$HOST_CODENAME" != "noble" ]; then
|
||||
SCRIPT_PATH="$(readlink -f "$0")"
|
||||
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
|
||||
mkdir -p "$OUT_DIR_ABS"
|
||||
# Wheels go to their own mount so PIP_DIR is honoured inside the container
|
||||
# (the in-container default would be the repo's pip-wheels/).
|
||||
PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)"
|
||||
|
||||
docker run --rm -i \
|
||||
-v "$REPO_DIR:/repo" \
|
||||
-v "$OUT_DIR_ABS:/out" \
|
||||
-v "$PIP_DIR_HOST:/pipout" \
|
||||
-e IN_DOCKER=1 \
|
||||
-e PIP_DIR=/pipout \
|
||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
||||
-e BLANCCO_KVER="${BLANCCO_KVER:-}" \
|
||||
-w /repo \
|
||||
ubuntu:24.04 \
|
||||
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out"
|
||||
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc"
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
@@ -85,7 +93,8 @@ PLAYBOOK_PACKAGES=(
|
||||
linux-generic-hwe-24.04
|
||||
"linux-image-${BLANCCO_KVER}"
|
||||
"linux-modules-${BLANCCO_KVER}"
|
||||
"linux-modules-extra-${BLANCCO_KVER}"
|
||||
# No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi
|
||||
# etc.) in linux-modules, and modules-extra-7.0.* does not exist.
|
||||
)
|
||||
|
||||
# Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.)
|
||||
@@ -114,9 +123,13 @@ sudo apt-get update -qq
|
||||
|
||||
# Simulate install to find all dependencies
|
||||
echo "[2/4] Resolving dependencies..."
|
||||
EXPLICIT_DEPS=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1 \
|
||||
| grep "^Inst " \
|
||||
| awk '{print $2}')
|
||||
SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || {
|
||||
# e.g. a package name that does not exist in noble: say which.
|
||||
echo "ERROR: apt could not resolve the package list:" >&2
|
||||
echo "$SIM" | grep -E "^(E|W):" >&2
|
||||
exit 1
|
||||
}
|
||||
EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}')
|
||||
|
||||
# ALSO pull every package that would upgrade in a dist-upgrade. This is
|
||||
# critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit
|
||||
@@ -147,9 +160,15 @@ echo " $DEB_COUNT packages ($TOTAL_SIZE)"
|
||||
echo "[4/4] Downloading Python wheels for webapp..."
|
||||
# Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
PIP_DIR="$REPO_ROOT/pip-wheels"
|
||||
mkdir -p "$PIP_DIR"
|
||||
pip3 download -d "$PIP_DIR" flask lxml 2>&1 | tail -5
|
||||
PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}"
|
||||
# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
|
||||
# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
|
||||
# every wheel in it and refuses any wheel that is not pinned.
|
||||
find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR"
|
||||
pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \
|
||||
--only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
|
||||
--platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \
|
||||
2>&1 | tail -5
|
||||
|
||||
WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l)
|
||||
echo " $WHL_COUNT Python packages downloaded to pip-wheels/"
|
||||
|
||||
Reference in New Issue
Block a user