pxe-server/scripts/download-packages.sh

#!/bin/bash
#
# download-packages.sh - Build the offline package set for the PXE server
#
# The server installs from the Ubuntu 24.04 server ISO with no network
# (autoinstall/user-data), so the installed base is exactly the ISO's
# "ubuntu-server" image plus what the installer adds from the ISO's pool/.
# This script resolves against THAT base, not against whatever the build
# host has:
#
#   1. The ISO's default install source (squashfs layers) is unpacked into
#      a docker image, pxe-iso-base:<iso version> (cached).
#   2. In a container of that image, with the Ubuntu archive (noble,
#      noble-updates, noble-security) as the source, apt downloads:
#        - every update of an installed package (dist-upgrade closure)
#        - the explicit package list below, with its dependencies
#        - the newest version of every package in the ISO's pool/ (what the
#          installer may add: kernel, bootloader, openssh-server...) and
#          the dependencies those versions need
#      and writes Packages (an apt index) and explicit.list next to the
#      debs, so the set is an apt repository.
#   3. A second container of the base image, with NO network, resolves the
#      install exactly as the server will (offline-apt-install.sh
#      --simulate). An unsatisfiable set fails the build here.
#   4. The webapp's pinned, hashed wheels go to pip-wheels/.
#
# On the server, autoinstall's late-commands run
#   offline-apt-install.sh /mnt/cidata/packages
# which upgrades every installed package that the set has a newer version
# of and installs explicit.list, all from the local repo only.
#
# Needs docker (no sudo; the user must be in the docker group), xorriso and
# internet access to the Ubuntu archive and PyPI.
#
# Usage:
#   ./download-packages.sh --iso ubuntu-24.04.x-live-server-amd64.iso [output_directory]
#   ISO=... ./download-packages.sh [output_directory]
#
# Default output: ./offline-packages/ (its old .debs are removed first).
# Wheels go to PIP_DIR (default: <repo>/pip-wheels).

set -euo pipefail

SCRIPT_PATH="$(readlink -f "$0")"
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"

# Kernel the Blancco UEFI path boots. The default is read from the playbook
# var blancco_kver (the one source; see its comment for the other places).
# The playbook builds vmlinuz-ubuntu-70, kexec-initrd-70.img and
# kmod-<kver>.tar.gz from this kernel's /boot and /lib/modules, and the
# Canonical-signed image is what Ubuntu's shim starts with Secure Boot on.
# The versioned packages pin it; the HWE metapackage keeps the box on the
# HWE line. Override: BLANCCO_KVER=7.0.0-40-generic ./download-packages.sh
# (then set blancco_kver to match, or the playbook will not find it).
if [ -z "${BLANCCO_KVER:-}" ]; then
    BLANCCO_KVER=$(sed -n 's/^[[:space:]]*blancco_kver:[[:space:]]*"\([^"]*\)".*/\1/p' \
        "$REPO_DIR/playbook/pxe_server_setup.yml" | head -1)
fi
[ -n "$BLANCCO_KVER" ] || { echo "ERROR: no blancco_kver in playbook/pxe_server_setup.yml (set BLANCCO_KVER)" >&2; exit 1; }

# Packages installed by the Ansible playbook (pxe_server_setup.yml)
PLAYBOOK_PACKAGES=(
    ansible
    dnsmasq
    apache2
    samba
    unzip
    ufw
    cron
    wimtools
    p7zip-full
    grub-efi-amd64-bin
    grub-common
    conntrack
    networkd-dispatcher   # carrier hook (pxe-nic-carrier); in the base, pinned here anyway
    busybox-static
    zstd
    cpio
    # Updates pages in the webapp (iPXE signature check, Dell pack zips, downloads)
    sbsigntool
    zip
    curl
    # Blancco with Secure Boot on: shimx64.efi.signed + the HWE kernel
    shim-signed
    linux-generic-hwe-24.04
    "linux-image-${BLANCCO_KVER}"
    "linux-modules-${BLANCCO_KVER}"
    # No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi
    # etc.) in linux-modules, and modules-extra-7.0.* does not exist.
)

# Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.)
AUTOINSTALL_PACKAGES=(
    network-manager
    wpasupplicant
    wireless-tools
    linux-firmware
    firmware-sof-signed
)

ALL_PACKAGES=("${PLAYBOOK_PACKAGES[@]}" "${AUTOINSTALL_PACKAGES[@]}")

# ---------------------------------------------------------------------------
# Inside the base container (internal; started by the host part below)
# ---------------------------------------------------------------------------
if [ "${IN_BASE:-}" = "1" ]; then
    export DEBIAN_FRONTEND=noninteractive
    OUT=/out
    ARCH=/tmp/arch
    mkdir -p "$ARCH/partial"

    # The ISO's sources point at the archive (and maybe the cdrom); use the
    # archive pockets only.
    rm -f /etc/apt/sources.list /etc/apt/sources.list.d/*
    cat >/etc/apt/sources.list.d/ubuntu.sources <<'SRC'
Types: deb
URIs: http://archive.ubuntu.com/ubuntu/
Suites: noble noble-updates
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg

Types: deb
URIs: http://security.ubuntu.com/ubuntu/
Suites: noble-security
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
SRC
    echo "[base] apt-get update"
    apt-get update -qq

    GET=(apt-get -o "Dir::Cache::archives=$ARCH" -o APT::Sandbox::User=root)

    # 1+2: dist-upgrade closure plus the explicit list, as one transaction
    # (the same one offline-apt-install.sh runs on the server).
    DU=$(apt-get -s dist-upgrade)
    if grep -q 'kept back' <<<"$DU"; then
        echo "ERROR: dist-upgrade keeps packages back on the ISO base:" >&2
        echo "$DU" >&2
        exit 1
    fi
    mapfile -t UPGRADES < <(awk '/^Inst /{print $2}' <<<"$DU")
    echo "[base] ${#UPGRADES[@]} installed packages have updates"
    if ! SIM=$(apt-get -s install "${ALL_PACKAGES[@]}" "${UPGRADES[@]}" 2>&1); then
        echo "ERROR: apt could not resolve the package list on the ISO base:" >&2
        grep -E "^(E|W):| : Depends" <<<"$SIM" >&2 || echo "$SIM" >&2
        exit 1
    fi
    echo "[base] downloading the install + upgrade transaction"
    "${GET[@]}" -y -q --download-only install "${ALL_PACKAGES[@]}" "${UPGRADES[@]}" >/tmp/dl.log 2>&1 \
        || { echo "ERROR: download failed:" >&2; tail -30 /tmp/dl.log >&2; exit 1; }

    # 3: newest version of every pool package the installer may add, with
    # the dependencies that version needs on the base. Versioned kernel ABI
    # packages (linux-*-6.8.0-71*) are reached through their metapackages.
    POOL_EXTRA=()
    SKIPPED=()
    while read -r name; do
        [ -n "$name" ] || continue
        case "$name" in
            linux-*-[0-9]*.[0-9]*.[0-9]*-[0-9]*) continue ;;
            # Driver packages for ubuntu-drivers (NVIDIA): never on this box.
            *nvidia*) continue ;;
        esac
        if PS=$(apt-get -s install "$name" 2>&1); then
            mapfile -t -O "${#POOL_EXTRA[@]}" POOL_EXTRA < <(awk '/^Inst /{print $2}' <<<"$PS")
        else
            SKIPPED+=("$name")
        fi
    done < /meta/pool.txt
    mapfile -t POOL_EXTRA < <(printf '%s\n' "${POOL_EXTRA[@]}" | sort -u)
    echo "[base] pool packages: $(wc -l </meta/pool.txt) names, closure ${#POOL_EXTRA[@]} packages"
    [ ${#SKIPPED[@]} -eq 0 ] || echo "[base] WARNING: pool packages not installable on the base, skipped: ${SKIPPED[*]}"
    (cd "$ARCH" && apt-get -q download "${POOL_EXTRA[@]}" >/tmp/pool-dl.log 2>&1) \
        || { echo "ERROR: pool download failed:" >&2; tail -30 /tmp/pool-dl.log >&2; exit 1; }

    # Several versions of one package (download + pool): keep the newest.
    find "$OUT" -maxdepth 1 \( -name '*.deb' -o -name 'Packages*' -o -name explicit.list \) -delete
    cp "$ARCH"/*.deb "$OUT"/
    python3 - "$OUT" <<'PY'
import glob, os, subprocess, sys
best = {}
for p in glob.glob(os.path.join(sys.argv[1], "*.deb")):
    n, v, a = os.path.basename(p)[:-4].split("_")
    v = v.replace("%3a", ":")
    k = (n, a)
    if k in best:
        old_v, old_p = best[k]
        if subprocess.run(["dpkg", "--compare-versions", v, "gt", old_v]).returncode == 0:
            os.remove(old_p); best[k] = (v, p)
        else:
            os.remove(p)
    else:
        best[k] = (v, p)
PY
    printf '%s\n' "${ALL_PACKAGES[@]}" >"$OUT/explicit.list"
    # Uncompressed index only: apt's file: method takes it as is.
    (cd "$OUT" && apt-ftparchive packages . >Packages 2>/dev/null)
    echo "[base] $(find "$OUT" -maxdepth 1 -name '*.deb' | wc -l) debs, $(du -sh "$OUT" | cut -f1)"
    chown -R "${HOST_UID:-0}:${HOST_GID:-0}" "$OUT"
    exit 0
fi

# ---------------------------------------------------------------------------
# Host part
# ---------------------------------------------------------------------------
ISO="${ISO:-}"
POSITIONAL=()
while [ $# -gt 0 ]; do
    case "$1" in
        --iso) ISO="$2"; shift 2 ;;
        -h|--help) sed -n '2,45p' "$SCRIPT_PATH"; exit 0 ;;
        *) POSITIONAL+=("$1"); shift ;;
    esac
done
OUT_DIR="${POSITIONAL[0]:-./offline-packages}"
mkdir -p "$OUT_DIR"
OUT_DIR_ABS="$(cd "$OUT_DIR" && pwd)"
PIP_DIR_HOST="${PIP_DIR:-$REPO_DIR/pip-wheels}"
mkdir -p "$PIP_DIR_HOST"
PIP_DIR_HOST="$(cd "$PIP_DIR_HOST" && pwd)"

[ -n "$ISO" ] || { echo "ERROR: give the server ISO: --iso ubuntu-24.04.x-live-server-amd64.iso (the same one build-usb.sh uses)" >&2; exit 1; }
[ -f "$ISO" ] || { echo "ERROR: ISO not found: $ISO" >&2; exit 1; }
for t in docker xorriso; do
    command -v "$t" >/dev/null || { echo "ERROR: $t not installed" >&2; exit 1; }
done

META=$(mktemp -d "${TMPDIR:-/tmp}/dlpkg.XXXXXX")
trap 'rm -rf "$META"' EXIT

echo "============================================"
echo "Offline package set for the PXE server"
echo "============================================"
echo "ISO     : $ISO"
echo "Output  : $OUT_DIR_ABS"
echo "Wheels  : $PIP_DIR_HOST"
echo ""

# --- 1. base image from the ISO's default install source ---
VOLID=$(xorriso -indev "$ISO" -pvd_info 2>/dev/null | sed -n 's/^Volume Id *: *//p' | head -1)
TAG="pxe-iso-base:$(tr 'A-Z ' 'a-z-' <<<"$VOLID" | tr -cd 'a-z0-9.-')"
xorriso -osirrox on -indev "$ISO" -extract /casper/install-sources.yaml "$META/install-sources.yaml" >/dev/null 2>&1
# Default source: the entry with "default: true"; a layered path a.b.squashfs
# is layer a.squashfs under a.b.squashfs.
SRC_PATH=$(awk '/^- /{d=0} /default: true/{d=1} /^  path:/{p=$2} /^  type:/{ if (d) {print p; exit} }' "$META/install-sources.yaml")
[ -n "$SRC_PATH" ] || { echo "ERROR: no default source in casper/install-sources.yaml" >&2; exit 1; }
LAYERS=()
stem="${SRC_PATH%.squashfs}"
acc=""
IFS=. read -r -a parts <<<"$stem"
for p in "${parts[@]}"; do
    acc="${acc:+$acc.}$p"
    LAYERS+=("$acc.squashfs")
done
echo "[1/4] Base: $VOLID, source $SRC_PATH (layers: ${LAYERS[*]}) -> $TAG"

xorriso -indev "$ISO" -find /pool -type f -name '*.deb' 2>/dev/null \
    | sed "s|.*/||; s|'\$||; s|_.*||" | sort -u >"$META/pool.txt"

if docker image inspect "$TAG" >/dev/null 2>&1; then
    echo "    image $TAG exists (docker rmi $TAG to rebuild)"
else
    for l in "${LAYERS[@]}"; do
        xorriso -osirrox on -indev "$ISO" -extract "/casper/$l" "$META/$l" >/dev/null 2>&1
    done
    # Unpack the layers as root in a throwaway container (keeps ownership),
    # apply overlayfs whiteouts, stream the tree into docker import.
    docker run --rm -i -v "$META:/meta:ro" ubuntu:24.04 bash -euc '
        apt-get update -qq >/dev/null && apt-get install -y -qq squashfs-tools >/dev/null
        mkdir /r
        first=1
        for l in '"${LAYERS[*]}"'; do
            if [ $first = 1 ]; then
                unsquashfs -q -no-xattrs -f -d /r "/meta/$l" >/dev/null; first=0
            else
                # Whiteouts (char devices) = files deleted in this layer.
                # The container may not mknod, so list them, delete those
                # paths from the tree and extract the layer without them.
                unsquashfs -lls "/meta/$l" | awk '"'"'$1 ~ /^c/ {sub(/^.*squashfs-root/, ""); print}'"'"' >/wo
                while read -r w; do rm -rf "/r$w"; done </wo
                rm -rf /l
                if [ -s /wo ]; then
                    unsquashfs -q -no-xattrs -d /l -excludes "/meta/$l" $(cat /wo) >/dev/null
                else
                    unsquashfs -q -no-xattrs -d /l "/meta/$l" >/dev/null
                fi
                cp -a /l/. /r/
            fi
        done
        tar -C /r -c .' >"$META/base.tar"
    # Import only a complete tree: a failed unpack must not leave an image.
    docker import "$META/base.tar" "$TAG" >/dev/null
    rm -f "$META/base.tar"
    docker run --rm "$TAG" test -x /usr/bin/apt-get \
        || { docker rmi "$TAG" >/dev/null; echo "ERROR: base image $TAG is incomplete" >&2; exit 1; }
    echo "    built $TAG"
fi
for l in "${LAYERS[@]}"; do rm -f "$META/$l"; done

# --- 2. resolve + download against the base ---
echo "[2/4] Resolving and downloading on the ISO base (this takes a while)..."
docker run --rm \
    -v "$REPO_DIR:/repo:ro" \
    -v "$OUT_DIR_ABS:/out" \
    -v "$META:/meta:ro" \
    -e IN_BASE=1 -e BLANCCO_KVER="$BLANCCO_KVER" \
    -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
    "$TAG" bash /repo/scripts/download-packages.sh

# The ISO this set was resolved against; build-usb.sh refuses another one.
printf '%s\n' "$VOLID" >"$OUT_DIR_ABS/iso-volid"

# --- 3. offline check: resolve exactly as the server will, no network ---
echo "[3/4] Offline check on a fresh base container (no network)..."
docker run --rm --network none \
    -v "$REPO_DIR:/repo:ro" -v "$OUT_DIR_ABS:/out:ro" \
    "$TAG" bash /repo/scripts/offline-apt-install.sh --simulate /out

# --- 4. pinned webapp wheels ---
echo "[4/4] Downloading Python wheels for the webapp..."
# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
# every wheel in it and refuses any wheel that is not pinned. Only a folder
# of wheels is emptied: a PIP_DIR pointing anywhere else is refused.
NOT_WHEELS=$(find "$PIP_DIR_HOST" -mindepth 1 \( ! -name '*.whl' -o ! -type f \) -print -quit)
if [ -n "$NOT_WHEELS" ]; then
    echo "ERROR: $PIP_DIR_HOST holds more than *.whl files (e.g. $NOT_WHEELS); refusing to empty it." >&2
    echo "       Point PIP_DIR at a wheels-only folder." >&2
    exit 1
fi
find "$PIP_DIR_HOST" -mindepth 1 -maxdepth 1 -name '*.whl' -type f -delete
docker run --rm -v "$REPO_DIR:/repo:ro" -v "$PIP_DIR_HOST:/pipout" \
    -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" ubuntu:24.04 bash -euc '
    apt-get update -qq >/dev/null
    apt-get install -y -qq --no-install-recommends python3-pip ca-certificates >/dev/null
    pip3 download -q -d /pipout --require-hashes -r /repo/webapp/requirements.txt \
        --only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
        --platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64
    chown -R "$HOST_UID:$HOST_GID" /pipout'

DEB_COUNT=$(find "$OUT_DIR_ABS" -maxdepth 1 -name '*.deb' | wc -l)
WHL_COUNT=$(find "$PIP_DIR_HOST" -maxdepth 1 \( -name '*.whl' -o -name '*.tar.gz' \) | wc -l)
echo ""
echo "============================================"
echo "Download complete!"
echo "============================================"
echo "  .deb packages: $DEB_COUNT ($(du -sh "$OUT_DIR_ABS" | cut -f1)) in $OUT_DIR_ABS/ (apt repo: Packages, explicit.list)"
echo "  Python wheels: $WHL_COUNT in $PIP_DIR_HOST/"
echo "  Check them:    scripts/check-updates.py --debs $OUT_DIR_ABS"
echo ""