bcd91b488a
- autoinstall: dead mirror + fallback offline-install, so the installed base is exactly the ISO's; late-commands run offline-apt-install.sh on CIDATA packages/ (local apt repo, one transaction, errors fail the install, log /var/log/offline-apt-install.log); first boot no longer runs dpkg -i; real archive sources written back at the end. - download-packages.sh --iso: unpacks the ISO's default install source into a docker image (pxe-iso-base:<version>) and downloads there: all updates of the base, the explicit list (zip included) and the newest pool packages with their closure; writes Packages + explicit.list and checks the set offline in a fresh base container (--network none). - offline-apt-install.sh: new; also used by that check (--simulate). - build-usb.sh: ships Packages, explicit.list and the script; refuses a set without them. - tests/vm: pxetest-client-nosb (Secure Boot off, e1000e, SATA), one client at a time.
334 lines
14 KiB
Bash
Executable File
334 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# download-packages.sh - Build the offline package set for the PXE server
|
|
#
|
|
# The server installs from the Ubuntu 24.04 server ISO with no network
|
|
# (autoinstall/user-data), so the installed base is exactly the ISO's
|
|
# "ubuntu-server" image plus what the installer adds from the ISO's pool/.
|
|
# This script resolves against THAT base, not against whatever the build
|
|
# host has:
|
|
#
|
|
# 1. The ISO's default install source (squashfs layers) is unpacked into
|
|
# a docker image, pxe-iso-base:<iso version> (cached).
|
|
# 2. In a container of that image, with the Ubuntu archive (noble,
|
|
# noble-updates, noble-security) as the source, apt downloads:
|
|
# - every update of an installed package (dist-upgrade closure)
|
|
# - the explicit package list below, with its dependencies
|
|
# - the newest version of every package in the ISO's pool/ (what the
|
|
# installer may add: kernel, bootloader, openssh-server...) and
|
|
# the dependencies those versions need
|
|
# and writes Packages (an apt index) and explicit.list next to the
|
|
# debs, so the set is an apt repository.
|
|
# 3. A second container of the base image, with NO network, resolves the
|
|
# install exactly as the server will (offline-apt-install.sh
|
|
# --simulate). An unsatisfiable set fails the build here.
|
|
# 4. The webapp's pinned, hashed wheels go to pip-wheels/.
|
|
#
|
|
# On the server, autoinstall's late-commands run
|
|
# offline-apt-install.sh /mnt/cidata/packages
|
|
# which upgrades every installed package that the set has a newer version
|
|
# of and installs explicit.list, all from the local repo only.
|
|
#
|
|
# Needs docker (no sudo; the user must be in the docker group), xorriso and
|
|
# internet access to the Ubuntu archive and PyPI.
|
|
#
|
|
# Usage:
|
|
# ./download-packages.sh --iso ubuntu-24.04.x-live-server-amd64.iso [output_directory]
|
|
# ISO=... ./download-packages.sh [output_directory]
|
|
#
|
|
# Default output: ./offline-packages/ (its old .debs are removed first).
|
|
# Wheels go to PIP_DIR (default: <repo>/pip-wheels).
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_PATH="$(readlink -f "$0")"
|
|
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
|
|
|
|
# Kernel the Blancco UEFI path boots (playbook var blancco_kver; keep the two
|
|
# in step). The playbook builds vmlinuz-ubuntu-70, kexec-initrd-70.img and
|
|
# kmod-<kver>.tar.gz from this kernel's /boot and /lib/modules, and the
|
|
# Canonical-signed image is what Ubuntu's shim starts with Secure Boot on.
|
|
# The versioned packages pin it; the HWE metapackage keeps the box on the
|
|
# HWE line. Override: BLANCCO_KVER=7.0.0-40-generic ./download-packages.sh
|
|
BLANCCO_KVER="${BLANCCO_KVER:-7.0.0-34-generic}"
|
|
|
|
# Packages installed by the Ansible playbook (pxe_server_setup.yml)
|
|
PLAYBOOK_PACKAGES=(
|
|
ansible
|
|
dnsmasq
|
|
apache2
|
|
samba
|
|
unzip
|
|
ufw
|
|
cron
|
|
wimtools
|
|
p7zip-full
|
|
grub-efi-amd64-bin
|
|
grub-common
|
|
conntrack
|
|
busybox-static
|
|
zstd
|
|
cpio
|
|
# Updates pages in the webapp (iPXE signature check, Dell pack zips, downloads)
|
|
sbsigntool
|
|
zip
|
|
curl
|
|
# Blancco with Secure Boot on: shimx64.efi.signed + the HWE kernel
|
|
shim-signed
|
|
linux-generic-hwe-24.04
|
|
"linux-image-${BLANCCO_KVER}"
|
|
"linux-modules-${BLANCCO_KVER}"
|
|
# No linux-modules-extra: from 7.0 Ubuntu ships those modules (iwlwifi
|
|
# etc.) in linux-modules, and modules-extra-7.0.* does not exist.
|
|
)
|
|
|
|
# Packages installed during autoinstall late-commands (NetworkManager, WiFi, etc.)
|
|
AUTOINSTALL_PACKAGES=(
|
|
network-manager
|
|
wpasupplicant
|
|
wireless-tools
|
|
linux-firmware
|
|
firmware-sof-signed
|
|
)
|
|
|
|
ALL_PACKAGES=("${PLAYBOOK_PACKAGES[@]}" "${AUTOINSTALL_PACKAGES[@]}")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Inside the base container (internal; started by the host part below)
|
|
# ---------------------------------------------------------------------------
|
|
if [ "${IN_BASE:-}" = "1" ]; then
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
OUT=/out
|
|
ARCH=/tmp/arch
|
|
mkdir -p "$ARCH/partial"
|
|
|
|
# The ISO's sources point at the archive (and maybe the cdrom); use the
|
|
# archive pockets only.
|
|
rm -f /etc/apt/sources.list /etc/apt/sources.list.d/*
|
|
cat >/etc/apt/sources.list.d/ubuntu.sources <<'SRC'
|
|
Types: deb
|
|
URIs: http://archive.ubuntu.com/ubuntu/
|
|
Suites: noble noble-updates
|
|
Components: main restricted universe multiverse
|
|
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
|
|
|
Types: deb
|
|
URIs: http://security.ubuntu.com/ubuntu/
|
|
Suites: noble-security
|
|
Components: main restricted universe multiverse
|
|
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
|
SRC
|
|
echo "[base] apt-get update"
|
|
apt-get update -qq
|
|
|
|
GET=(apt-get -o "Dir::Cache::archives=$ARCH" -o APT::Sandbox::User=root)
|
|
|
|
# 1+2: dist-upgrade closure plus the explicit list, as one transaction
|
|
# (the same one offline-apt-install.sh runs on the server).
|
|
DU=$(apt-get -s dist-upgrade)
|
|
if grep -q 'kept back' <<<"$DU"; then
|
|
echo "ERROR: dist-upgrade keeps packages back on the ISO base:" >&2
|
|
echo "$DU" >&2
|
|
exit 1
|
|
fi
|
|
mapfile -t UPGRADES < <(awk '/^Inst /{print $2}' <<<"$DU")
|
|
echo "[base] ${#UPGRADES[@]} installed packages have updates"
|
|
if ! SIM=$(apt-get -s install "${ALL_PACKAGES[@]}" "${UPGRADES[@]}" 2>&1); then
|
|
echo "ERROR: apt could not resolve the package list on the ISO base:" >&2
|
|
grep -E "^(E|W):| : Depends" <<<"$SIM" >&2 || echo "$SIM" >&2
|
|
exit 1
|
|
fi
|
|
echo "[base] downloading the install + upgrade transaction"
|
|
"${GET[@]}" -y -q --download-only install "${ALL_PACKAGES[@]}" "${UPGRADES[@]}" >/tmp/dl.log 2>&1 \
|
|
|| { echo "ERROR: download failed:" >&2; tail -30 /tmp/dl.log >&2; exit 1; }
|
|
|
|
# 3: newest version of every pool package the installer may add, with
|
|
# the dependencies that version needs on the base. Versioned kernel ABI
|
|
# packages (linux-*-6.8.0-71*) are reached through their metapackages.
|
|
POOL_EXTRA=()
|
|
SKIPPED=()
|
|
while read -r name; do
|
|
[ -n "$name" ] || continue
|
|
case "$name" in
|
|
linux-*-[0-9]*.[0-9]*.[0-9]*-[0-9]*) continue ;;
|
|
# Driver packages for ubuntu-drivers (NVIDIA): never on this box.
|
|
*nvidia*) continue ;;
|
|
esac
|
|
if PS=$(apt-get -s install "$name" 2>&1); then
|
|
mapfile -t -O "${#POOL_EXTRA[@]}" POOL_EXTRA < <(awk '/^Inst /{print $2}' <<<"$PS")
|
|
else
|
|
SKIPPED+=("$name")
|
|
fi
|
|
done < /meta/pool.txt
|
|
mapfile -t POOL_EXTRA < <(printf '%s\n' "${POOL_EXTRA[@]}" | sort -u)
|
|
echo "[base] pool packages: $(wc -l </meta/pool.txt) names, closure ${#POOL_EXTRA[@]} packages"
|
|
[ ${#SKIPPED[@]} -eq 0 ] || echo "[base] WARNING: pool packages not installable on the base, skipped: ${SKIPPED[*]}"
|
|
(cd "$ARCH" && apt-get -q download "${POOL_EXTRA[@]}" >/tmp/pool-dl.log 2>&1) \
|
|
|| { echo "ERROR: pool download failed:" >&2; tail -30 /tmp/pool-dl.log >&2; exit 1; }
|
|
|
|
# Several versions of one package (download + pool): keep the newest.
|
|
find "$OUT" -maxdepth 1 \( -name '*.deb' -o -name 'Packages*' -o -name explicit.list \) -delete
|
|
cp "$ARCH"/*.deb "$OUT"/
|
|
python3 - "$OUT" <<'PY'
|
|
import glob, os, subprocess, sys
|
|
best = {}
|
|
for p in glob.glob(os.path.join(sys.argv[1], "*.deb")):
|
|
n, v, a = os.path.basename(p)[:-4].split("_")
|
|
v = v.replace("%3a", ":")
|
|
k = (n, a)
|
|
if k in best:
|
|
old_v, old_p = best[k]
|
|
if subprocess.run(["dpkg", "--compare-versions", v, "gt", old_v]).returncode == 0:
|
|
os.remove(old_p); best[k] = (v, p)
|
|
else:
|
|
os.remove(p)
|
|
else:
|
|
best[k] = (v, p)
|
|
PY
|
|
printf '%s\n' "${ALL_PACKAGES[@]}" >"$OUT/explicit.list"
|
|
# Uncompressed index only: apt's file: method takes it as is.
|
|
(cd "$OUT" && apt-ftparchive packages . >Packages 2>/dev/null)
|
|
echo "[base] $(find "$OUT" -maxdepth 1 -name '*.deb' | wc -l) debs, $(du -sh "$OUT" | cut -f1)"
|
|
chown -R "${HOST_UID:-0}:${HOST_GID:-0}" "$OUT"
|
|
exit 0
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Host part
|
|
# ---------------------------------------------------------------------------
|
|
ISO="${ISO:-}"
|
|
POSITIONAL=()
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--iso) ISO="$2"; shift 2 ;;
|
|
-h|--help) sed -n '2,45p' "$SCRIPT_PATH"; exit 0 ;;
|
|
*) POSITIONAL+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
OUT_DIR="${POSITIONAL[0]:-./offline-packages}"
|
|
mkdir -p "$OUT_DIR"
|
|
OUT_DIR_ABS="$(cd "$OUT_DIR" && pwd)"
|
|
PIP_DIR_HOST="${PIP_DIR:-$REPO_DIR/pip-wheels}"
|
|
mkdir -p "$PIP_DIR_HOST"
|
|
PIP_DIR_HOST="$(cd "$PIP_DIR_HOST" && pwd)"
|
|
|
|
[ -n "$ISO" ] || { echo "ERROR: give the server ISO: --iso ubuntu-24.04.x-live-server-amd64.iso (the same one build-usb.sh uses)" >&2; exit 1; }
|
|
[ -f "$ISO" ] || { echo "ERROR: ISO not found: $ISO" >&2; exit 1; }
|
|
for t in docker xorriso; do
|
|
command -v "$t" >/dev/null || { echo "ERROR: $t not installed" >&2; exit 1; }
|
|
done
|
|
|
|
META=$(mktemp -d "${TMPDIR:-/tmp}/dlpkg.XXXXXX")
|
|
trap 'rm -rf "$META"' EXIT
|
|
|
|
echo "============================================"
|
|
echo "Offline package set for the PXE server"
|
|
echo "============================================"
|
|
echo "ISO : $ISO"
|
|
echo "Output : $OUT_DIR_ABS"
|
|
echo "Wheels : $PIP_DIR_HOST"
|
|
echo ""
|
|
|
|
# --- 1. base image from the ISO's default install source ---
|
|
VOLID=$(xorriso -indev "$ISO" -pvd_info 2>/dev/null | sed -n 's/^Volume Id *: *//p' | head -1)
|
|
TAG="pxe-iso-base:$(tr 'A-Z ' 'a-z-' <<<"$VOLID" | tr -cd 'a-z0-9.-')"
|
|
xorriso -osirrox on -indev "$ISO" -extract /casper/install-sources.yaml "$META/install-sources.yaml" >/dev/null 2>&1
|
|
# Default source: the entry with "default: true"; a layered path a.b.squashfs
|
|
# is layer a.squashfs under a.b.squashfs.
|
|
SRC_PATH=$(awk '/^- /{d=0} /default: true/{d=1} /^ path:/{p=$2} /^ type:/{ if (d) {print p; exit} }' "$META/install-sources.yaml")
|
|
[ -n "$SRC_PATH" ] || { echo "ERROR: no default source in casper/install-sources.yaml" >&2; exit 1; }
|
|
LAYERS=()
|
|
stem="${SRC_PATH%.squashfs}"
|
|
acc=""
|
|
IFS=. read -r -a parts <<<"$stem"
|
|
for p in "${parts[@]}"; do
|
|
acc="${acc:+$acc.}$p"
|
|
LAYERS+=("$acc.squashfs")
|
|
done
|
|
echo "[1/4] Base: $VOLID, source $SRC_PATH (layers: ${LAYERS[*]}) -> $TAG"
|
|
|
|
xorriso -indev "$ISO" -find /pool -type f -name '*.deb' 2>/dev/null \
|
|
| sed "s|.*/||; s|'\$||; s|_.*||" | sort -u >"$META/pool.txt"
|
|
|
|
if docker image inspect "$TAG" >/dev/null 2>&1; then
|
|
echo " image $TAG exists (docker rmi $TAG to rebuild)"
|
|
else
|
|
for l in "${LAYERS[@]}"; do
|
|
xorriso -osirrox on -indev "$ISO" -extract "/casper/$l" "$META/$l" >/dev/null 2>&1
|
|
done
|
|
# Unpack the layers as root in a throwaway container (keeps ownership),
|
|
# apply overlayfs whiteouts, stream the tree into docker import.
|
|
docker run --rm -i -v "$META:/meta:ro" ubuntu:24.04 bash -euc '
|
|
apt-get update -qq >/dev/null && apt-get install -y -qq squashfs-tools >/dev/null
|
|
mkdir /r
|
|
first=1
|
|
for l in '"${LAYERS[*]}"'; do
|
|
if [ $first = 1 ]; then
|
|
unsquashfs -q -no-xattrs -f -d /r "/meta/$l" >/dev/null; first=0
|
|
else
|
|
# Whiteouts (char devices) = files deleted in this layer.
|
|
# The container may not mknod, so list them, delete those
|
|
# paths from the tree and extract the layer without them.
|
|
unsquashfs -lls "/meta/$l" | awk '"'"'$1 ~ /^c/ {sub(/^.*squashfs-root/, ""); print}'"'"' >/wo
|
|
while read -r w; do rm -rf "/r$w"; done </wo
|
|
rm -rf /l
|
|
if [ -s /wo ]; then
|
|
unsquashfs -q -no-xattrs -d /l -excludes "/meta/$l" $(cat /wo) >/dev/null
|
|
else
|
|
unsquashfs -q -no-xattrs -d /l "/meta/$l" >/dev/null
|
|
fi
|
|
cp -a /l/. /r/
|
|
fi
|
|
done
|
|
tar -C /r -c .' >"$META/base.tar"
|
|
# Import only a complete tree: a failed unpack must not leave an image.
|
|
docker import "$META/base.tar" "$TAG" >/dev/null
|
|
rm -f "$META/base.tar"
|
|
docker run --rm "$TAG" test -x /usr/bin/apt-get \
|
|
|| { docker rmi "$TAG" >/dev/null; echo "ERROR: base image $TAG is incomplete" >&2; exit 1; }
|
|
echo " built $TAG"
|
|
fi
|
|
for l in "${LAYERS[@]}"; do rm -f "$META/$l"; done
|
|
|
|
# --- 2. resolve + download against the base ---
|
|
echo "[2/4] Resolving and downloading on the ISO base (this takes a while)..."
|
|
docker run --rm \
|
|
-v "$REPO_DIR:/repo:ro" \
|
|
-v "$OUT_DIR_ABS:/out" \
|
|
-v "$META:/meta:ro" \
|
|
-e IN_BASE=1 -e BLANCCO_KVER="$BLANCCO_KVER" \
|
|
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
|
"$TAG" bash /repo/scripts/download-packages.sh
|
|
|
|
# --- 3. offline check: resolve exactly as the server will, no network ---
|
|
echo "[3/4] Offline check on a fresh base container (no network)..."
|
|
docker run --rm --network none \
|
|
-v "$REPO_DIR:/repo:ro" -v "$OUT_DIR_ABS:/out:ro" \
|
|
"$TAG" bash /repo/scripts/offline-apt-install.sh --simulate /out
|
|
|
|
# --- 4. pinned webapp wheels ---
|
|
echo "[4/4] Downloading Python wheels for the webapp..."
|
|
# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
|
|
# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
|
|
# every wheel in it and refuses any wheel that is not pinned.
|
|
find "$PIP_DIR_HOST" -mindepth 1 -delete
|
|
docker run --rm -v "$REPO_DIR:/repo:ro" -v "$PIP_DIR_HOST:/pipout" \
|
|
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" ubuntu:24.04 bash -euc '
|
|
apt-get update -qq >/dev/null
|
|
apt-get install -y -qq --no-install-recommends python3-pip ca-certificates >/dev/null
|
|
pip3 download -q -d /pipout --require-hashes -r /repo/webapp/requirements.txt \
|
|
--only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
|
|
--platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64
|
|
chown -R "$HOST_UID:$HOST_GID" /pipout'
|
|
|
|
DEB_COUNT=$(find "$OUT_DIR_ABS" -maxdepth 1 -name '*.deb' | wc -l)
|
|
WHL_COUNT=$(find "$PIP_DIR_HOST" -maxdepth 1 \( -name '*.whl' -o -name '*.tar.gz' \) | wc -l)
|
|
echo ""
|
|
echo "============================================"
|
|
echo "Download complete!"
|
|
echo "============================================"
|
|
echo " .deb packages: $DEB_COUNT ($(du -sh "$OUT_DIR_ABS" | cut -f1)) in $OUT_DIR_ABS/ (apt repo: Packages, explicit.list)"
|
|
echo " Python wheels: $WHL_COUNT in $PIP_DIR_HOST/"
|
|
echo " Check them: scripts/check-updates.py --debs $OUT_DIR_ABS"
|
|
echo ""
|