Offline install that lands fully patched: resolve against the ISO's own base
- autoinstall: dead mirror + fallback offline-install, so the installed base is exactly the ISO's; late-commands run offline-apt-install.sh on CIDATA packages/ (local apt repo, one transaction, errors fail the install, log /var/log/offline-apt-install.log); first boot no longer runs dpkg -i; real archive sources written back at the end. - download-packages.sh --iso: unpacks the ISO's default install source into a docker image (pxe-iso-base:<version>) and downloads there: all updates of the base, the explicit list (zip included) and the newest pool packages with their closure; writes Packages + explicit.list and checks the set offline in a fresh base container (--network none). - offline-apt-install.sh: new; also used by that check (--simulate). - build-usb.sh: ships Packages, explicit.list and the script; refuses a set without them. - tests/vm: pxetest-client-nosb (Secure Boot off, e1000e, SATA), one client at a time.
This commit is contained in:
cproudlock
@@ -170,6 +170,17 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
# The installer patches and installs from packages/ as an apt repo
|
||||
# (autoinstall late-commands -> offline-apt-install.sh): it needs the index
|
||||
# and the explicit list that download-packages.sh writes with the debs.
|
||||
for f in "$OFFLINE_PKG_DIR/Packages" "$OFFLINE_PKG_DIR/explicit.list" "$REPO_ROOT/scripts/offline-apt-install.sh"; do
|
||||
if [ ! -f "$f" ]; then
|
||||
echo "ERROR: $f not found. Build the offline set with"
|
||||
echo " scripts/download-packages.sh --iso $ISO_PATH $OFFLINE_PKG_DIR"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$SITE_VARS" ] && [ ! -f "$SITE_VARS" ]; then
|
||||
echo "ERROR: --site-vars file not found: $SITE_VARS"
|
||||
exit 1
|
||||
@@ -487,16 +498,17 @@ echo "[5/6] Copying autoinstall config, packages, and playbook to CIDATA..."
|
||||
# Copy cloud-init files
|
||||
t_cp "" "$AUTOINSTALL_DIR/user-data" "$AUTOINSTALL_DIR/meta-data"
|
||||
|
||||
# Copy offline .deb packages into packages/ subdirectory
|
||||
# Copy the offline package set (an apt repo: debs + Packages + explicit.list)
|
||||
# into packages/, and the script the installer runs on it to the root.
|
||||
t_mkdir packages
|
||||
t_cp packages "$OFFLINE_PKG_DIR/Packages" "$OFFLINE_PKG_DIR/explicit.list"
|
||||
DEB_COUNT=0
|
||||
if [ -d "$OFFLINE_PKG_DIR" ] && compgen -G "$OFFLINE_PKG_DIR/*.deb" >/dev/null; then
|
||||
if compgen -G "$OFFLINE_PKG_DIR/*.deb" >/dev/null; then
|
||||
t_cp packages "$OFFLINE_PKG_DIR"/*.deb
|
||||
DEB_COUNT=$(find "$OFFLINE_PKG_DIR" -maxdepth 1 -name '*.deb' | wc -l)
|
||||
else
|
||||
echo " WARNING: no .deb packages in $OFFLINE_PKG_DIR (run download-packages.sh)"
|
||||
fi
|
||||
echo " Copied $DEB_COUNT .deb packages to packages/"
|
||||
t_cp "" "$REPO_ROOT/scripts/offline-apt-install.sh"
|
||||
echo " Copied $DEB_COUNT .deb packages + Packages + explicit.list to packages/"
|
||||
|
||||
# Copy playbook directory
|
||||
t_cp "" "$PLAYBOOK_DIR"
|
||||
|
||||
@@ -1,63 +1,48 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# download-packages.sh - Download all .deb packages needed for offline PXE server setup
|
||||
# download-packages.sh - Build the offline package set for the PXE server
|
||||
#
|
||||
# The PXE server installs Ubuntu 24.04 (Noble), so all packages MUST come from the
|
||||
# 24.04 archive. If this script is run on a non-24.04 host (e.g. Zorin 17 / 22.04),
|
||||
# it auto-spawns an Ubuntu 24.04 docker container to do the download.
|
||||
# The server installs from the Ubuntu 24.04 server ISO with no network
|
||||
# (autoinstall/user-data), so the installed base is exactly the ISO's
|
||||
# "ubuntu-server" image plus what the installer adds from the ISO's pool/.
|
||||
# This script resolves against THAT base, not against whatever the build
|
||||
# host has:
|
||||
#
|
||||
# 1. The ISO's default install source (squashfs layers) is unpacked into
|
||||
# a docker image, pxe-iso-base:<iso version> (cached).
|
||||
# 2. In a container of that image, with the Ubuntu archive (noble,
|
||||
# noble-updates, noble-security) as the source, apt downloads:
|
||||
# - every update of an installed package (dist-upgrade closure)
|
||||
# - the explicit package list below, with its dependencies
|
||||
# - the newest version of every package in the ISO's pool/ (what the
|
||||
# installer may add: kernel, bootloader, openssh-server...) and
|
||||
# the dependencies those versions need
|
||||
# and writes Packages (an apt index) and explicit.list next to the
|
||||
# debs, so the set is an apt repository.
|
||||
# 3. A second container of the base image, with NO network, resolves the
|
||||
# install exactly as the server will (offline-apt-install.sh
|
||||
# --simulate). An unsatisfiable set fails the build here.
|
||||
# 4. The webapp's pinned, hashed wheels go to pip-wheels/.
|
||||
#
|
||||
# On the server, autoinstall's late-commands run
|
||||
# offline-apt-install.sh /mnt/cidata/packages
|
||||
# which upgrades every installed package that the set has a newer version
|
||||
# of and installs explicit.list, all from the local repo only.
|
||||
#
|
||||
# Needs docker (no sudo; the user must be in the docker group), xorriso and
|
||||
# internet access to the Ubuntu archive and PyPI.
|
||||
#
|
||||
# Usage:
|
||||
# ./download-packages.sh [output_directory]
|
||||
# ./download-packages.sh --iso ubuntu-24.04.x-live-server-amd64.iso [output_directory]
|
||||
# ISO=... ./download-packages.sh [output_directory]
|
||||
#
|
||||
# Default output: ./offline-packages/
|
||||
# Default output: ./offline-packages/ (its old .debs are removed first).
|
||||
# Wheels go to PIP_DIR (default: <repo>/pip-wheels).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
OUT_DIR="${1:-./offline-packages}"
|
||||
OUT_DIR_ABS="$(cd "$(dirname "$OUT_DIR")" 2>/dev/null && pwd)/$(basename "$OUT_DIR")"
|
||||
|
||||
# Detect host Ubuntu codename. Run inside the container if not Noble (24.04).
|
||||
HOST_CODENAME="$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}")"
|
||||
|
||||
# USE_DOCKER=1 forces the container even on a noble host: no sudo needed,
|
||||
# and the host's own apt state is left alone.
|
||||
if [ "${IN_DOCKER:-}" != "1" ] && { [ "$HOST_CODENAME" != "noble" ] || [ "${USE_DOCKER:-}" = "1" ]; }; then
|
||||
echo "Host is '$HOST_CODENAME'; downloading inside an ubuntu:24.04 container."
|
||||
echo "Re-running inside ubuntu:24.04 docker container..."
|
||||
echo ""
|
||||
|
||||
if ! command -v docker >/dev/null; then
|
||||
echo "ERROR: docker not installed. Install docker or run on a real Ubuntu 24.04 host."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_PATH="$(readlink -f "$0")"
|
||||
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
|
||||
mkdir -p "$OUT_DIR_ABS"
|
||||
# Wheels go to their own mount so PIP_DIR is honoured inside the container
|
||||
# (the in-container default would be the repo's pip-wheels/).
|
||||
PIP_DIR_HOST="$(mkdir -p "${PIP_DIR:-$REPO_DIR/pip-wheels}" && cd "${PIP_DIR:-$REPO_DIR/pip-wheels}" && pwd)"
|
||||
|
||||
docker run --rm -i \
|
||||
-v "$REPO_DIR:/repo" \
|
||||
-v "$OUT_DIR_ABS:/out" \
|
||||
-v "$PIP_DIR_HOST:/pipout" \
|
||||
-e IN_DOCKER=1 \
|
||||
-e PIP_DIR=/pipout \
|
||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
||||
-e BLANCCO_KVER="${BLANCCO_KVER:-}" \
|
||||
-w /repo \
|
||||
ubuntu:24.04 \
|
||||
bash -c "apt-get update -qq && apt-get install -y --no-install-recommends sudo python3-pip python3-setuptools python3-wheel ca-certificates >/dev/null && /repo/scripts/download-packages.sh /out; rc=\$?; chown -R \$HOST_UID:\$HOST_GID /out /pipout; exit \$rc"
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo "Container build complete. Files in: $OUT_DIR_ABS"
|
||||
echo "============================================"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
SCRIPT_PATH="$(readlink -f "$0")"
|
||||
REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)"
|
||||
|
||||
# Kernel the Blancco UEFI path boots (playbook var blancco_kver; keep the two
|
||||
# in step). The playbook builds vmlinuz-ubuntu-70, kexec-initrd-70.img and
|
||||
@@ -108,75 +93,241 @@ AUTOINSTALL_PACKAGES=(
|
||||
|
||||
ALL_PACKAGES=("${PLAYBOOK_PACKAGES[@]}" "${AUTOINSTALL_PACKAGES[@]}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inside the base container (internal; started by the host part below)
|
||||
# ---------------------------------------------------------------------------
|
||||
if [ "${IN_BASE:-}" = "1" ]; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
OUT=/out
|
||||
ARCH=/tmp/arch
|
||||
mkdir -p "$ARCH/partial"
|
||||
|
||||
# The ISO's sources point at the archive (and maybe the cdrom); use the
|
||||
# archive pockets only.
|
||||
rm -f /etc/apt/sources.list /etc/apt/sources.list.d/*
|
||||
cat >/etc/apt/sources.list.d/ubuntu.sources <<'SRC'
|
||||
Types: deb
|
||||
URIs: http://archive.ubuntu.com/ubuntu/
|
||||
Suites: noble noble-updates
|
||||
Components: main restricted universe multiverse
|
||||
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
||||
|
||||
Types: deb
|
||||
URIs: http://security.ubuntu.com/ubuntu/
|
||||
Suites: noble-security
|
||||
Components: main restricted universe multiverse
|
||||
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
||||
SRC
|
||||
echo "[base] apt-get update"
|
||||
apt-get update -qq
|
||||
|
||||
GET=(apt-get -o "Dir::Cache::archives=$ARCH" -o APT::Sandbox::User=root)
|
||||
|
||||
# 1+2: dist-upgrade closure plus the explicit list, as one transaction
|
||||
# (the same one offline-apt-install.sh runs on the server).
|
||||
DU=$(apt-get -s dist-upgrade)
|
||||
if grep -q 'kept back' <<<"$DU"; then
|
||||
echo "ERROR: dist-upgrade keeps packages back on the ISO base:" >&2
|
||||
echo "$DU" >&2
|
||||
exit 1
|
||||
fi
|
||||
mapfile -t UPGRADES < <(awk '/^Inst /{print $2}' <<<"$DU")
|
||||
echo "[base] ${#UPGRADES[@]} installed packages have updates"
|
||||
if ! SIM=$(apt-get -s install "${ALL_PACKAGES[@]}" "${UPGRADES[@]}" 2>&1); then
|
||||
echo "ERROR: apt could not resolve the package list on the ISO base:" >&2
|
||||
grep -E "^(E|W):| : Depends" <<<"$SIM" >&2 || echo "$SIM" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[base] downloading the install + upgrade transaction"
|
||||
"${GET[@]}" -y -q --download-only install "${ALL_PACKAGES[@]}" "${UPGRADES[@]}" >/tmp/dl.log 2>&1 \
|
||||
|| { echo "ERROR: download failed:" >&2; tail -30 /tmp/dl.log >&2; exit 1; }
|
||||
|
||||
# 3: newest version of every pool package the installer may add, with
|
||||
# the dependencies that version needs on the base. Versioned kernel ABI
|
||||
# packages (linux-*-6.8.0-71*) are reached through their metapackages.
|
||||
POOL_EXTRA=()
|
||||
SKIPPED=()
|
||||
while read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
case "$name" in
|
||||
linux-*-[0-9]*.[0-9]*.[0-9]*-[0-9]*) continue ;;
|
||||
# Driver packages for ubuntu-drivers (NVIDIA): never on this box.
|
||||
*nvidia*) continue ;;
|
||||
esac
|
||||
if PS=$(apt-get -s install "$name" 2>&1); then
|
||||
mapfile -t -O "${#POOL_EXTRA[@]}" POOL_EXTRA < <(awk '/^Inst /{print $2}' <<<"$PS")
|
||||
else
|
||||
SKIPPED+=("$name")
|
||||
fi
|
||||
done < /meta/pool.txt
|
||||
mapfile -t POOL_EXTRA < <(printf '%s\n' "${POOL_EXTRA[@]}" | sort -u)
|
||||
echo "[base] pool packages: $(wc -l </meta/pool.txt) names, closure ${#POOL_EXTRA[@]} packages"
|
||||
[ ${#SKIPPED[@]} -eq 0 ] || echo "[base] WARNING: pool packages not installable on the base, skipped: ${SKIPPED[*]}"
|
||||
(cd "$ARCH" && apt-get -q download "${POOL_EXTRA[@]}" >/tmp/pool-dl.log 2>&1) \
|
||||
|| { echo "ERROR: pool download failed:" >&2; tail -30 /tmp/pool-dl.log >&2; exit 1; }
|
||||
|
||||
# Several versions of one package (download + pool): keep the newest.
|
||||
find "$OUT" -maxdepth 1 \( -name '*.deb' -o -name 'Packages*' -o -name explicit.list \) -delete
|
||||
cp "$ARCH"/*.deb "$OUT"/
|
||||
python3 - "$OUT" <<'PY'
|
||||
import glob, os, subprocess, sys
|
||||
best = {}
|
||||
for p in glob.glob(os.path.join(sys.argv[1], "*.deb")):
|
||||
n, v, a = os.path.basename(p)[:-4].split("_")
|
||||
v = v.replace("%3a", ":")
|
||||
k = (n, a)
|
||||
if k in best:
|
||||
old_v, old_p = best[k]
|
||||
if subprocess.run(["dpkg", "--compare-versions", v, "gt", old_v]).returncode == 0:
|
||||
os.remove(old_p); best[k] = (v, p)
|
||||
else:
|
||||
os.remove(p)
|
||||
else:
|
||||
best[k] = (v, p)
|
||||
PY
|
||||
printf '%s\n' "${ALL_PACKAGES[@]}" >"$OUT/explicit.list"
|
||||
# Uncompressed index only: apt's file: method takes it as is.
|
||||
(cd "$OUT" && apt-ftparchive packages . >Packages 2>/dev/null)
|
||||
echo "[base] $(find "$OUT" -maxdepth 1 -name '*.deb' | wc -l) debs, $(du -sh "$OUT" | cut -f1)"
|
||||
chown -R "${HOST_UID:-0}:${HOST_GID:-0}" "$OUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Host part
|
||||
# ---------------------------------------------------------------------------
|
||||
ISO="${ISO:-}"
|
||||
POSITIONAL=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--iso) ISO="$2"; shift 2 ;;
|
||||
-h|--help) sed -n '2,45p' "$SCRIPT_PATH"; exit 0 ;;
|
||||
*) POSITIONAL+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
OUT_DIR="${POSITIONAL[0]:-./offline-packages}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
OUT_DIR_ABS="$(cd "$OUT_DIR" && pwd)"
|
||||
PIP_DIR_HOST="${PIP_DIR:-$REPO_DIR/pip-wheels}"
|
||||
mkdir -p "$PIP_DIR_HOST"
|
||||
PIP_DIR_HOST="$(cd "$PIP_DIR_HOST" && pwd)"
|
||||
|
||||
[ -n "$ISO" ] || { echo "ERROR: give the server ISO: --iso ubuntu-24.04.x-live-server-amd64.iso (the same one build-usb.sh uses)" >&2; exit 1; }
|
||||
[ -f "$ISO" ] || { echo "ERROR: ISO not found: $ISO" >&2; exit 1; }
|
||||
for t in docker xorriso; do
|
||||
command -v "$t" >/dev/null || { echo "ERROR: $t not installed" >&2; exit 1; }
|
||||
done
|
||||
|
||||
META=$(mktemp -d "${TMPDIR:-/tmp}/dlpkg.XXXXXX")
|
||||
trap 'rm -rf "$META"' EXIT
|
||||
|
||||
echo "============================================"
|
||||
echo "Offline Package Downloader (Ubuntu 24.04 noble)"
|
||||
echo "Offline package set for the PXE server"
|
||||
echo "============================================"
|
||||
echo "Output directory: $OUT_DIR"
|
||||
echo ""
|
||||
echo "Packages to resolve:"
|
||||
printf ' - %s\n' "${ALL_PACKAGES[@]}"
|
||||
echo "ISO : $ISO"
|
||||
echo "Output : $OUT_DIR_ABS"
|
||||
echo "Wheels : $PIP_DIR_HOST"
|
||||
echo ""
|
||||
|
||||
# Update package cache
|
||||
echo "[1/4] Updating package cache..."
|
||||
sudo apt-get update -qq
|
||||
# --- 1. base image from the ISO's default install source ---
|
||||
VOLID=$(xorriso -indev "$ISO" -pvd_info 2>/dev/null | sed -n 's/^Volume Id *: *//p' | head -1)
|
||||
TAG="pxe-iso-base:$(tr 'A-Z ' 'a-z-' <<<"$VOLID" | tr -cd 'a-z0-9.-')"
|
||||
xorriso -osirrox on -indev "$ISO" -extract /casper/install-sources.yaml "$META/install-sources.yaml" >/dev/null 2>&1
|
||||
# Default source: the entry with "default: true"; a layered path a.b.squashfs
|
||||
# is layer a.squashfs under a.b.squashfs.
|
||||
SRC_PATH=$(awk '/^- /{d=0} /default: true/{d=1} /^ path:/{p=$2} /^ type:/{ if (d) {print p; exit} }' "$META/install-sources.yaml")
|
||||
[ -n "$SRC_PATH" ] || { echo "ERROR: no default source in casper/install-sources.yaml" >&2; exit 1; }
|
||||
LAYERS=()
|
||||
stem="${SRC_PATH%.squashfs}"
|
||||
acc=""
|
||||
IFS=. read -r -a parts <<<"$stem"
|
||||
for p in "${parts[@]}"; do
|
||||
acc="${acc:+$acc.}$p"
|
||||
LAYERS+=("$acc.squashfs")
|
||||
done
|
||||
echo "[1/4] Base: $VOLID, source $SRC_PATH (layers: ${LAYERS[*]}) -> $TAG"
|
||||
|
||||
# Simulate install to find all dependencies
|
||||
echo "[2/4] Resolving dependencies..."
|
||||
SIM=$(apt-get install --simulate "${ALL_PACKAGES[@]}" 2>&1) || {
|
||||
# e.g. a package name that does not exist in noble: say which.
|
||||
echo "ERROR: apt could not resolve the package list:" >&2
|
||||
echo "$SIM" | grep -E "^(E|W):" >&2
|
||||
exit 1
|
||||
}
|
||||
EXPLICIT_DEPS=$(echo "$SIM" | grep "^Inst " | awk '{print $2}')
|
||||
xorriso -indev "$ISO" -find /pool -type f -name '*.deb' 2>/dev/null \
|
||||
| sed "s|.*/||; s|'\$||; s|_.*||" | sort -u >"$META/pool.txt"
|
||||
|
||||
# ALSO pull every package that would upgrade in a dist-upgrade. This is
|
||||
# critical: the Ubuntu ISO ships a point-in-time baseline, but our explicit
|
||||
# packages (from noble-updates) may depend on *newer* versions of ISO-baseline
|
||||
# packages (e.g. gnupg 17.4 needs matching gpgv 17.4). Without this, offline
|
||||
# install fails with dpkg "dependency problems" because transitive version
|
||||
# bumps aren't captured by --simulate on the explicit list.
|
||||
UPGRADE_DEPS=$(apt-get dist-upgrade --simulate 2>&1 \
|
||||
| grep "^Inst " \
|
||||
| awk '{print $2}')
|
||||
if docker image inspect "$TAG" >/dev/null 2>&1; then
|
||||
echo " image $TAG exists (docker rmi $TAG to rebuild)"
|
||||
else
|
||||
for l in "${LAYERS[@]}"; do
|
||||
xorriso -osirrox on -indev "$ISO" -extract "/casper/$l" "$META/$l" >/dev/null 2>&1
|
||||
done
|
||||
# Unpack the layers as root in a throwaway container (keeps ownership),
|
||||
# apply overlayfs whiteouts, stream the tree into docker import.
|
||||
docker run --rm -i -v "$META:/meta:ro" ubuntu:24.04 bash -euc '
|
||||
apt-get update -qq >/dev/null && apt-get install -y -qq squashfs-tools >/dev/null
|
||||
mkdir /r
|
||||
first=1
|
||||
for l in '"${LAYERS[*]}"'; do
|
||||
if [ $first = 1 ]; then
|
||||
unsquashfs -q -no-xattrs -f -d /r "/meta/$l" >/dev/null; first=0
|
||||
else
|
||||
# Whiteouts (char devices) = files deleted in this layer.
|
||||
# The container may not mknod, so list them, delete those
|
||||
# paths from the tree and extract the layer without them.
|
||||
unsquashfs -lls "/meta/$l" | awk '"'"'$1 ~ /^c/ {sub(/^.*squashfs-root/, ""); print}'"'"' >/wo
|
||||
while read -r w; do rm -rf "/r$w"; done </wo
|
||||
rm -rf /l
|
||||
if [ -s /wo ]; then
|
||||
unsquashfs -q -no-xattrs -d /l -excludes "/meta/$l" $(cat /wo) >/dev/null
|
||||
else
|
||||
unsquashfs -q -no-xattrs -d /l "/meta/$l" >/dev/null
|
||||
fi
|
||||
cp -a /l/. /r/
|
||||
fi
|
||||
done
|
||||
tar -C /r -c .' >"$META/base.tar"
|
||||
# Import only a complete tree: a failed unpack must not leave an image.
|
||||
docker import "$META/base.tar" "$TAG" >/dev/null
|
||||
rm -f "$META/base.tar"
|
||||
docker run --rm "$TAG" test -x /usr/bin/apt-get \
|
||||
|| { docker rmi "$TAG" >/dev/null; echo "ERROR: base image $TAG is incomplete" >&2; exit 1; }
|
||||
echo " built $TAG"
|
||||
fi
|
||||
for l in "${LAYERS[@]}"; do rm -f "$META/$l"; done
|
||||
|
||||
DEPS=$(printf '%s\n%s\n' "$EXPLICIT_DEPS" "$UPGRADE_DEPS" | sort -u | grep -v '^$')
|
||||
# --- 2. resolve + download against the base ---
|
||||
echo "[2/4] Resolving and downloading on the ISO base (this takes a while)..."
|
||||
docker run --rm \
|
||||
-v "$REPO_DIR:/repo:ro" \
|
||||
-v "$OUT_DIR_ABS:/out" \
|
||||
-v "$META:/meta:ro" \
|
||||
-e IN_BASE=1 -e BLANCCO_KVER="$BLANCCO_KVER" \
|
||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
||||
"$TAG" bash /repo/scripts/download-packages.sh
|
||||
|
||||
DEP_COUNT=$(echo "$DEPS" | wc -l)
|
||||
echo " Found $DEP_COUNT packages (explicit + baseline upgrades)"
|
||||
# --- 3. offline check: resolve exactly as the server will, no network ---
|
||||
echo "[3/4] Offline check on a fresh base container (no network)..."
|
||||
docker run --rm --network none \
|
||||
-v "$REPO_DIR:/repo:ro" -v "$OUT_DIR_ABS:/out:ro" \
|
||||
"$TAG" bash /repo/scripts/offline-apt-install.sh --simulate /out
|
||||
|
||||
# Download all packages
|
||||
echo "[3/4] Downloading .deb packages to $OUT_DIR..."
|
||||
cd "$OUT_DIR"
|
||||
apt-get download $DEPS 2>&1 | tail -5
|
||||
|
||||
DEB_COUNT=$(ls -1 *.deb 2>/dev/null | wc -l)
|
||||
TOTAL_SIZE=$(du -sh . | cut -f1)
|
||||
|
||||
echo " $DEB_COUNT packages ($TOTAL_SIZE)"
|
||||
|
||||
# Download pip wheels for Flask webapp (offline install)
|
||||
echo "[4/4] Downloading Python wheels for webapp..."
|
||||
# Place pip-wheels next to the script (or /repo when in docker), not next to OUT_DIR
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
PIP_DIR="${PIP_DIR:-$REPO_ROOT/pip-wheels}"
|
||||
# --- 4. pinned webapp wheels ---
|
||||
echo "[4/4] Downloading Python wheels for the webapp..."
|
||||
# Exactly the pinned, hashed set in webapp/requirements.txt, for the server's
|
||||
# Python 3.12 on x86_64. The folder is emptied first: the playbook installs
|
||||
# every wheel in it and refuses any wheel that is not pinned.
|
||||
find "$PIP_DIR" -mindepth 1 -delete 2>/dev/null; mkdir -p "$PIP_DIR"
|
||||
pip3 download -d "$PIP_DIR" --require-hashes -r "$REPO_ROOT/webapp/requirements.txt" \
|
||||
--only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
|
||||
--platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \
|
||||
2>&1 | tail -5
|
||||
|
||||
WHL_COUNT=$(ls -1 "$PIP_DIR"/*.whl "$PIP_DIR"/*.tar.gz 2>/dev/null | wc -l)
|
||||
echo " $WHL_COUNT Python packages downloaded to pip-wheels/"
|
||||
find "$PIP_DIR_HOST" -mindepth 1 -delete
|
||||
docker run --rm -v "$REPO_DIR:/repo:ro" -v "$PIP_DIR_HOST:/pipout" \
|
||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" ubuntu:24.04 bash -euc '
|
||||
apt-get update -qq >/dev/null
|
||||
apt-get install -y -qq --no-install-recommends python3-pip ca-certificates >/dev/null
|
||||
pip3 download -q -d /pipout --require-hashes -r /repo/webapp/requirements.txt \
|
||||
--only-binary=:all: --no-deps --python-version 3.12 --implementation cp --abi cp312 \
|
||||
--platform manylinux_2_28_x86_64 --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64
|
||||
chown -R "$HOST_UID:$HOST_GID" /pipout'
|
||||
|
||||
DEB_COUNT=$(find "$OUT_DIR_ABS" -maxdepth 1 -name '*.deb' | wc -l)
|
||||
WHL_COUNT=$(find "$PIP_DIR_HOST" -maxdepth 1 \( -name '*.whl' -o -name '*.tar.gz' \) | wc -l)
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo "Download complete!"
|
||||
echo "============================================"
|
||||
echo " .deb packages: $DEB_COUNT ($TOTAL_SIZE) in $OUT_DIR/"
|
||||
echo " Python wheels: $WHL_COUNT in $PIP_DIR/"
|
||||
echo " .deb packages: $DEB_COUNT ($(du -sh "$OUT_DIR_ABS" | cut -f1)) in $OUT_DIR_ABS/ (apt repo: Packages, explicit.list)"
|
||||
echo " Python wheels: $WHL_COUNT in $PIP_DIR_HOST/"
|
||||
echo " Check them: scripts/check-updates.py --debs $OUT_DIR_ABS"
|
||||
echo ""
|
||||
|
||||
85
scripts/offline-apt-install.sh
Executable file
85
scripts/offline-apt-install.sh
Executable file
@@ -0,0 +1,85 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# offline-apt-install.sh - Install and patch from the offline package set
|
||||
#
|
||||
# Uses DIR (offline-packages/ as built by download-packages.sh: the .debs,
|
||||
# a Packages index and explicit.list) as the ONLY apt source:
|
||||
# 1. every installed package that DIR has a newer version of is upgraded
|
||||
# 2. every package in DIR/explicit.list is installed
|
||||
# in one apt transaction, then checks that nothing is left to upgrade.
|
||||
# The system's own sources.list and lists are not read or changed.
|
||||
#
|
||||
# Any apt failure stops the script with a non-zero exit and apt's message.
|
||||
#
|
||||
# Usage:
|
||||
# offline-apt-install.sh DIR install (as root)
|
||||
# offline-apt-install.sh --simulate DIR resolve only (apt-get -s)
|
||||
#
|
||||
# Runs in the autoinstall late-commands (curtin in-target) and in
|
||||
# download-packages.sh's offline check of the bundle.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SIMULATE=0
|
||||
if [ "${1:-}" = "--simulate" ]; then SIMULATE=1; shift; fi
|
||||
DIR="${1:?usage: $0 [--simulate] DIR}"
|
||||
DIR="$(cd "$DIR" && pwd)"
|
||||
|
||||
for f in Packages explicit.list; do
|
||||
[ -f "$DIR/$f" ] || { echo "offline-apt: ERROR: $DIR/$f missing (rebuild with download-packages.sh)" >&2; exit 1; }
|
||||
done
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
mkdir -p "$WORK/lists/partial" "$WORK/parts"
|
||||
echo "deb [trusted=yes] file:$DIR ./" >"$WORK/sources.list"
|
||||
|
||||
APT=(apt-get
|
||||
-o "Dir::Etc::SourceList=$WORK/sources.list"
|
||||
-o "Dir::Etc::SourceParts=$WORK/parts"
|
||||
-o "Dir::State::Lists=$WORK/lists"
|
||||
-o "APT::Sandbox::User=root"
|
||||
-o "Dpkg::Options::=--force-confdef"
|
||||
-o "Dpkg::Options::=--force-confold")
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
echo "offline-apt: source $DIR ($(find "$DIR" -maxdepth 1 -name '*.deb' | wc -l) debs)"
|
||||
"${APT[@]}" update
|
||||
|
||||
mapfile -t EXPLICIT < <(grep -v '^[[:space:]]*\(#\|$\)' "$DIR/explicit.list")
|
||||
DU=$("${APT[@]}" -s dist-upgrade)
|
||||
if grep -q 'kept back' <<<"$DU"; then
|
||||
echo "offline-apt: ERROR: upgrades kept back (missing dependencies in $DIR):" >&2
|
||||
echo "$DU" >&2
|
||||
exit 1
|
||||
fi
|
||||
mapfile -t UPGRADES < <(awk '/^Inst /{print $2}' <<<"$DU")
|
||||
echo "offline-apt: ${#UPGRADES[@]} upgrades, ${#EXPLICIT[@]} explicit packages"
|
||||
|
||||
if [ "$SIMULATE" -eq 1 ]; then
|
||||
OUT=$("${APT[@]}" -s install "${EXPLICIT[@]}" "${UPGRADES[@]}")
|
||||
echo "offline-apt: simulated: $(grep -c '^Inst ' <<<"$OUT") packages to unpack, $(grep -c '^Conf ' <<<"$OUT") to configure"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
"${APT[@]}" -y install "${EXPLICIT[@]}" "${UPGRADES[@]}"
|
||||
|
||||
AFTER=$("${APT[@]}" -s dist-upgrade)
|
||||
LEFT=$(awk '/^Inst /{print $2}' <<<"$AFTER")
|
||||
if grep -q 'kept back' <<<"$AFTER"; then
|
||||
echo "offline-apt: ERROR: upgrades kept back (missing dependencies in $DIR):" >&2
|
||||
echo "$AFTER" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$LEFT" ]; then
|
||||
echo "offline-apt: ERROR: still upgradable from $DIR after install:" >&2
|
||||
echo "$LEFT" >&2
|
||||
exit 1
|
||||
fi
|
||||
BROKEN=$(dpkg --audit || true)
|
||||
if [ -n "$BROKEN" ]; then
|
||||
echo "offline-apt: ERROR: dpkg --audit reports:" >&2
|
||||
echo "$BROKEN" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "offline-apt: done, nothing left to upgrade from the offline set"
|
||||
Reference in New Issue
Block a user