diff --git a/.githooks/pre-commit b/.githooks/pre-commit index ed7bc32..3a8ae69 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -5,7 +5,7 @@ # git config core.hooksPath .githooks set -euo pipefail ROOT="$(git rev-parse --show-toplevel)" -mapfile -t FILES < <(git diff --cached --name-only --diff-filter=ACM | grep -iE '(^|/)(Flat)?Unattend[^/]*\.xml$' || true) +mapfile -t FILES < <(git diff --cached --name-only --diff-filter=ACMR | grep -iE '(^|/)(Flat)?Unattend[^/]*\.xml$' || true) [ ${#FILES[@]} -eq 0 ] && exit 0 TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT STAGED=() diff --git a/README.md b/README.md index 071da42..8cf72dd 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ pxe-server/ ├── bios-staging/ # Dell BIOS update binaries (gitignored) ├── scripts/ # Build, deploy, and helper scripts │ ├── build-usb.sh # Builds the installer USB (2-partition) -│ ├── build-proxmox-iso.sh # Builds self-contained Proxmox installer ISO +│ ├── build-proxmox-iso.sh # UNSUPPORTED (use build-usb.sh --image) │ ├── prepare-boot-tools.sh # Extracts and patches boot tool files │ ├── download-packages.sh # Downloads offline .debs + pip wheels │ ├── download-drivers.py # Downloads Dell drivers directly from dell.com @@ -193,34 +193,13 @@ pxe-server/ ## Proxmox Deployment -A single ISO can be built for deploying the PXE server in a Proxmox VM: - -### Build the ISO - -```bash -# Prerequisites (on build workstation) -sudo apt install xorriso p7zip-full - -# Build the installer ISO -./scripts/build-proxmox-iso.sh /path/to/ubuntu-24.04-live-server-amd64.iso -``` - -This creates `pxe-server-proxmox.iso` containing the Ubuntu installer, autoinstall config, all offline packages, the Ansible playbook, webapp, and boot tools. - -### Deploy on Proxmox - -1. Upload `pxe-server-proxmox.iso` to Proxmox storage (Datacenter -> Storage -> ISO Images) -2. Create a new VM: - - **OS:** Linux 6.x kernel - - **BIOS:** OVMF (UEFI) or SeaBIOS - - **Memory:** 4096 MB - - **CPU:** 2+ cores - - **Disk:** 40+ GB (VirtIO SCSI) - - **Network:** Bridge connected to your isolated PXE network -3. Attach the ISO as CD-ROM and start the VM -4. Ubuntu auto-installs with zero interaction (~10-15 minutes) -5. After reboot, first-boot configures all PXE services automatically -6. Access the web interface at `http://172.16.9.1:9009` +`scripts/build-proxmox-iso.sh` is no longer supported: it was a second, +drifted copy of the installer. Build a disk image with +`scripts/build-usb.sh --site-vars site-vars.yml --image pxe-usb.img auto UBUNTU.iso` +and import it on the Proxmox host (`qm importdisk pxe-usb.img +`), attached after the system disk in the boot order. The VM needs +two NICs: one on an isolated imaging bridge (no other DHCP server) and one +on the corp bridge; their MACs go in `site-vars.yml`. ### Import WinPE Images diff --git a/SETUP.md b/SETUP.md index 11963fb..f2bdebc 100644 --- a/SETUP.md +++ b/SETUP.md @@ -89,22 +89,29 @@ Extracts boot files for Blancco, Clonezilla, and Memtest86+ into the `boot-tools ### Step 3: Build the USB ```bash -# Basic — server only (import WinPE images later) -sudo ./scripts/build-usb.sh /dev/sdX /path/to/ubuntu-24.04-live-server-amd64.iso +# Basic — server only (import WinPE images later). site-vars.yml holds the +# two NIC MACs (copy playbook/site-vars.example.yml); --site-vars or +# --no-site-vars is required. +sudo ./scripts/build-usb.sh --site-vars site-vars.yml /dev/sdX /path/to/ubuntu-24.04-live-server-amd64.iso -# With WinPE images bundled (single USB, larger drive needed) -sudo ./scripts/build-usb.sh /dev/sdX /path/to/ubuntu-24.04.iso /path/to/winpe-images +# With WinPE images bundled (single USB, larger drive needed; no file may +# be 4 GiB or more, FAT32) +sudo ./scripts/build-usb.sh --site-vars site-vars.yml /dev/sdX /path/to/ubuntu-24.04.iso /path/to/winpe-images # Same layout into a disk-image file, no root (dd it to a stick later, or -# boot it in a VM: see tests/vm/README.md). SIZE is e.g. 16G or "auto". -./scripts/build-usb.sh --image pxe-usb.img auto /path/to/ubuntu-24.04.iso +# boot it in a VM: see tests/vm/README.md; on Proxmox: qm importdisk). +# SIZE is e.g. 16G or "auto". +./scripts/build-usb.sh --site-vars site-vars.yml --image pxe-usb.img auto /path/to/ubuntu-24.04.iso ``` Options: `--payload-root DIR` takes offline-packages/, pip-wheels/, boot-files/, boot-tools/, enrollment/, bios-staging/ and drivers-staging/ from another checkout; `--skip-enrollment`, `--skip-bios`, `--skip-drivers`, `--skip-boot-tools` leave those out; `--yes` skips the -prompt. The ISO volume label and El Torito load sizes are read from the +prompt. The build refuses payloads that do not match their pins: the iPXE +files against the playbook's `ipxe_*_sha256`, each wheel against its own +package's hashes in `webapp/requirements.txt` (no PyPI fallback), and the +ISO against the volume id `offline-packages/iso-volid` was resolved for. The ISO volume label and El Torito load sizes are read from the ISO, so any 24.04.x point release works. This creates a bootable USB with: @@ -116,13 +123,19 @@ This creates a bootable USB with: 1. Insert the USB into the target machine 2. Press F12 (or vendor boot key) and select the USB 3. Ubuntu auto-installs — no interaction needed -4. After reboot, the first-boot script installs all .deb packages and runs the Ansible playbook +4. After reboot, the first-boot script runs the Ansible playbook (the .debs + were installed by the installer); it pins the two NICs by MAC 5. PXE services (dnsmasq, Apache, Samba) are configured automatically 6. Flask webapp starts on port 9009 +7. If the playbook fails, `/var/lib/pxe-firstboot.failed` and a banner on + the console and at SSH login point at `/var/log/first-boot.log`; first + boot retries at the next reboot (5 attempts) ### Step 5: Connect to Isolated Network -Move the server's wired NIC to the isolated switch for PXE clients. +Both NICs are cabled before the install: the imaging NIC (`pxe_nic_mac`) +to the isolated switch for PXE clients, the corp NIC (`corp_nic_mac`) to +the GE network. Nothing needs to move after the install. ### Step 6: Import WinPE Content (if not bundled in Step 3) @@ -217,7 +230,7 @@ pxe-server/ ├── bios-staging/ # Dell BIOS update binaries (gitignored) ├── scripts/ # Build, deploy, and helper scripts │ ├── build-usb.sh # Builds the 2-partition installer USB -│ ├── build-proxmox-iso.sh # Builds self-contained Proxmox installer ISO +│ ├── build-proxmox-iso.sh # UNSUPPORTED (use build-usb.sh --image) │ ├── prepare-boot-tools.sh # Extracts/patches boot tools from ISOs │ ├── download-packages.sh # Downloads all offline packages │ ├── download-drivers.py # Downloads Dell drivers from dell.com diff --git a/autoinstall/user-data b/autoinstall/user-data index 1dbd766..6dc13d8 100644 --- a/autoinstall/user-data +++ b/autoinstall/user-data @@ -89,11 +89,38 @@ autoinstall: # Step 2: Disable networkd-wait-online (hangs on air-gapped network) - ln -sf /dev/null /target/etc/systemd/system/systemd-networkd-wait-online.service - # Step 3: Create first-boot.sh + # Step 3: Create first-boot.sh. rc.local runs it on every boot until the + # playbook succeeds (at most MAX attempts). A failure leaves + # /var/lib/pxe-firstboot.failed and a banner on the console (/etc/issue) + # and at SSH login (/etc/motd) pointing at /var/log/first-boot.log. - | curtin in-target --target=/target -- bash -c ' cat <<"EOF" > /opt/first-boot.sh #!/bin/bash + MAX=5 + STATE=/var/lib/pxe-firstboot + FAILED=/var/lib/pxe-firstboot.failed + mkdir -p "$STATE" + N=$(( $(cat "$STATE/attempts" 2>/dev/null || echo 0) + 1 )) + echo "$N" > "$STATE/attempts" + echo "=== first-boot attempt $N of $MAX, $(date -Is) ===" + banner_off() { + for f in /etc/issue /etc/motd; do + [ -f "$f" ] && sed -i "/^### PXE FIRST-BOOT BEGIN/,/^### PXE FIRST-BOOT END/d" "$f" + done + } + banner_on() { + banner_off + for f in /etc/issue /etc/motd; do + { + echo "### PXE FIRST-BOOT BEGIN" + echo "!!! PXE server setup (first-boot playbook) FAILED: $1" + echo "!!! Log: /var/log/first-boot.log Marker: $FAILED" + echo "!!! $2" + echo "### PXE FIRST-BOOT END" + } >> "$f" + done + } # Expand root LV to full disk BEFORE playbook (playbook copies ~130GB of drivers+PPKGs) ROOT_DEV=$(findmnt -n -o SOURCE /) ROOT_DISK=$(lsblk -n -o PKNAME "$(readlink -f "$ROOT_DEV")" | tail -1) @@ -104,28 +131,52 @@ autoinstall: pvresize "$PV_PART" 2>&1 || true fi lvextend -r -l +100%FREE /dev/mapper/ubuntu--vg-ubuntu--lv 2>&1 || true + RC=1 + WHY="CIDATA (the installer USB) not found" CIDATA_DEV=$(blkid -L CIDATA) if [ -n "$CIDATA_DEV" ]; then mkdir -p /mnt/usb - mount "$CIDATA_DEV" /mnt/usb - if [ -f /mnt/usb/playbook/pxe_server_setup.yml ]; then - cd /mnt/usb/playbook - ansible-playbook -i localhost, -c local pxe_server_setup.yml - cd / + if mount "$CIDATA_DEV" /mnt/usb; then + if [ -f /mnt/usb/playbook/pxe_server_setup.yml ]; then + cd /mnt/usb/playbook + ansible-playbook -i localhost, -c local pxe_server_setup.yml + RC=$? + WHY="ansible-playbook exit code $RC" + cd / + else + WHY="no playbook/pxe_server_setup.yml on CIDATA" + fi + umount /mnt/usb + else + WHY="could not mount CIDATA ($CIDATA_DEV)" fi - umount /mnt/usb fi - sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local + if [ "$RC" -eq 0 ]; then + echo "=== first-boot: playbook succeeded on attempt $N ===" + rm -f "$FAILED" + banner_off + sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local + exit 0 + fi + echo "=== first-boot: FAILED ($WHY), attempt $N of $MAX ===" + printf "attempt=%s of %s\nwhen=%s\nreason=%s\nlog=/var/log/first-boot.log\n" "$N" "$MAX" "$(date -Is)" "$WHY" > "$FAILED" + if [ "$N" -ge "$MAX" ]; then + sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local + banner_on "$WHY (attempt $N of $MAX)" "Gave up retrying. Fix it, then run: sudo /opt/first-boot.sh after rm $STATE/attempts" + else + banner_on "$WHY (attempt $N of $MAX)" "It runs again at the next boot (fix the cause first, e.g. playbook/site-vars.yml on the USB)." + fi + exit "$RC" EOF chmod +x /opt/first-boot.sh ' - # Step 4: Create rc.local + # Step 4: Create rc.local (the log is appended to, one block per attempt) - | curtin in-target --target=/target -- bash -c ' cat <<"EOF" > /etc/rc.local #!/bin/bash - /opt/first-boot.sh > /var/log/first-boot.log 2>&1 & + /opt/first-boot.sh >> /var/log/first-boot.log 2>&1 & exit 0 EOF chmod +x /etc/rc.local diff --git a/docs/SITE-CUSTOMIZATION.md b/docs/SITE-CUSTOMIZATION.md index a853884..27d09c7 100644 --- a/docs/SITE-CUSTOMIZATION.md +++ b/docs/SITE-CUSTOMIZATION.md @@ -37,6 +37,32 @@ netplan (names `pxe0` / `corp0`, overridable), dnsmasq, UFW and Samba. | NIC MACs / names | (required) / pxe0, corp0 | `playbook/site-vars.yml` (`pxe_nic_mac`, `corp_nic_mac`, `pxe_nic_name`, `corp_nic_name`) | | Hostname | pxeserver | `autoinstall/user-data` (identity.hostname) | +### Access from the GE corporate network + +SSH (password auth, the shared `pxe` password) and the webapp on 9009 (no +login, runs as root) are reachable on the corp NIC. By default from ALL of +GE, as before, so a re-run never locks anyone out; the play prints a loud +warning on every run while that is so. Set these in `playbook/site-vars.yml`: + +| Var | Default | Effect | +|--------------------------|---------|--------| +| `corp_admin_cidrs` | `[]` | List of admin subnets/hosts, e.g. `["10.134.48.0/24", "3.12.34.56/32"]`. When non-empty, UFW allows 22 and 9009 on `corp_nic_name` from these only and drops the any-source rules. The play refuses (nothing changed) if an entry is invalid or your own SSH session is not inside one of them. The imaging NIC keeps its rules. | +| `corp_ssh_password_auth` | `true` | `false` adds `/etc/ssh/sshd_config.d/10-pxe-corp-auth.conf`: `PasswordAuthentication no`, re-allowed by a `Match Address` block for `pxe_subnet` and localhost. Keys only from GE. The play refuses while `pxe` has no `~/.ssh/authorized_keys`, and removes the drop-in again if `sshd -t` rejects it. | +| `allow_disruptive` | `false` | Re-runs while imaging is active (SMB sessions on 445, or DHCP leases granted in the last `pxe_busy_window_min` = 30 min) do not bounce the NICs or restart dnsmasq/Apache/Samba; a needed network change stops the play. `-e allow_disruptive=true` overrides. | + +Recommended for a production box: `corp_admin_cidrs` set to the admin +desks' subnet, then (after `ssh-copy-id`) `corp_ssh_password_auth: false`. + +### Re-running the playbook over SSH (from the GE side) + +The network step refuses before changing anything when the NIC that +`pxe_nic_mac` names holds the default route, a DHCP lease, or the route to +your SSH client (swapped MACs or cables). When it does apply, it arms a +120 s auto-revert timer (`pxe-netplan-revert-*`), checks both names, the +imaging address and that the corp NIC gets its IPv4 back within 30 s, and +restores the previous netplan config AND NIC names on any failure. Backups: +`/var/backups/netplan-/`. + ### Identity and credentials | Value | Default | Where it lives | diff --git a/playbook/pxe-server-helpers/pxe-netplan-apply.sh b/playbook/pxe-server-helpers/pxe-netplan-apply.sh index ecc9a3c..0532346 100755 --- a/playbook/pxe-server-helpers/pxe-netplan-apply.sh +++ b/playbook/pxe-server-helpers/pxe-netplan-apply.sh @@ -2,43 +2,64 @@ # pxe-netplan-apply.sh - install the two-NIC netplan config without ever # leaving the box unreachable on the corporate NIC. # -# Usage: pxe-netplan-apply.sh STAGED_YAML PXE_NAME PXE_MAC CORP_NAME CORP_MAC +# Usage: +# pxe-netplan-apply.sh STAGED_YAML PXE_NAME PXE_MAC CORP_NAME CORP_MAC +# pxe-netplan-apply.sh --preflight STAGED_YAML PXE_NAME PXE_MAC CORP_NAME CORP_MAC +# pxe-netplan-apply.sh --revert BACKUP_DIR (the auto-revert timer) # -# Called by pxe_server_setup.yml. Steps: +# Env (set by pxe_server_setup.yml): +# PXE_SUBNET imaging LAN, e.g. 172.16.9.0/24 (SSH peers inside it may +# legitimately come in on the imaging NIC) +# PXE_SSH_CLIENT IP of the admin's SSH client ($SSH_CONNECTION field 1); +# every established sshd peer is checked as well +# PXE_NETPLAN_REVERT_AFTER seconds before the auto-revert fires (120) +# PXE_NETPLAN_CORP_WAIT seconds to wait for the corp IPv4 again (30) +# +# Steps (apply): # 1. validate STAGED_YAML in a scratch root (netplan generate --root-dir); # nothing on the box changes if it is invalid -# 2. stop here (prints UNCHANGED) if it is already installed, no other -# file configures wired NICs, and both NICs carry their stable names -# 3. back up /etc/netplan to /var/backups/netplan-/ -# 4. install it (other wired configs moved aside), netplan generate -# (restore on failure) -# 5. rename the NICs to their stable names (the .link files netplan -# writes do it on every later boot), netplan apply (restore on failure) -# 6. check the imaging address and the corp NIC (warning only: the corp -# cable may simply be unplugged) -# Prints CHANGED or UNCHANGED as the last line. Exit 1 = nothing applied -# (the previous config is back in place). +# 2. stop here (UNCHANGED) if it is already installed, no other file +# configures wired NICs, and both NICs carry their stable names +# 3. pre-flight: REFUSE (exit 1, nothing changed) when the NIC that +# PXE_MAC names holds the default route, a DHCP lease, or the route to +# an SSH client outside PXE_SUBNET. That NIC becomes static with no +# gateway, so any of these means it is the corp-cabled one (swapped +# MACs) and applying would cut the admin off. +# 4. back up /etc/netplan to /var/backups/netplan-/, record the +# current NIC names, and arm an auto-revert (systemd-run timer) that +# restores the backup unless step 8 cancels it +# 5. install it (other wired configs moved aside), netplan generate +# 6. rename the NICs to their stable names (the .link files netplan +# writes do it on every later boot) +# 7. netplan apply +# 8. check: both names present, the imaging NIC has its IPv4, the corp +# NIC has IPv4 again if it had one before (PXE_NETPLAN_CORP_WAIT s), +# SSH clients still route out of the corp side; then cancel the revert +# Any failure in 5-8 restores the backup AND the previous NIC names, then +# exits 1. Prints CHANGED / UNCHANGED (or NEEDED for --preflight) as the +# last line. set -u -if [ $# -ne 5 ]; then - echo "usage: $0 STAGED_YAML PXE_NAME PXE_MAC CORP_NAME CORP_MAC" >&2 - exit 2 -fi -STAGED=$1 -PXE_NAME=$2 -PXE_MAC=${3,,} -CORP_NAME=$4 -CORP_MAC=${5,,} NETPLAN_DIR=${NETPLAN_DIR:-/etc/netplan} +SYSFS_NET=${SYSFS_NET:-/sys/class/net} +BACKUP_ROOT=${BACKUP_ROOT:-/var/backups} +LEASE_DIR=${LEASE_DIR:-/run/systemd/netif/leases} +LOCK_FILE=${LOCK_FILE:-/run/pxe-netplan-apply.lock} +REVERT_AFTER=${PXE_NETPLAN_REVERT_AFTER:-120} +CORP_WAIT=${PXE_NETPLAN_CORP_WAIT:-30} +PXE_WAIT=${PXE_NETPLAN_PXE_WAIT:-20} TARGET=$NETPLAN_DIR/50-cloud-init.yaml +SELF=$(readlink -f "$0") log() { echo "pxe-netplan-apply: $*"; } +# --- NIC helpers ------------------------------------------------------------- + # Name of the physical NIC that has MAC $1 (empty if none). iface_of() { local d - for d in /sys/class/net/*; do + for d in "$SYSFS_NET"/*; do [ -e "$d/device" ] || continue if [ "$(cat "$d/address" 2>/dev/null)" = "$1" ]; then basename "$d" @@ -48,11 +69,159 @@ iface_of() { return 0 } +has_v4() { + ip -4 -o addr show dev "$1" scope global 2>/dev/null | grep -q ' inet ' +} + +default_route_devs() { + { ip -4 route show default; ip -6 route show default; } 2>/dev/null | + awk '{for (i = 1; i < NF; i++) if ($i == "dev") print $(i + 1)}' | sort -u +} + +# DHCP-assigned address (networkd and NetworkManager both flag those +# "dynamic"), or a networkd lease file for the link. +has_dhcp_lease() { + local idx + ip -o addr show dev "$1" 2>/dev/null | grep -qw dynamic && return 0 + idx=$(cat "$SYSFS_NET/$1/ifindex" 2>/dev/null) + [ -n "$idx" ] && [ -s "$LEASE_DIR/$idx" ] && return 0 + return 1 +} + +route_dev() { + ip -o route get "$1" 2>/dev/null | + awk '{for (i = 1; i < NF; i++) if ($i == "dev") {print $(i + 1); exit}}' +} + +ip2int() { + local a b c d + [[ "$1" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)$ ]] || return 1 + a=${BASH_REMATCH[1]} b=${BASH_REMATCH[2]} c=${BASH_REMATCH[3]} d=${BASH_REMATCH[4]} + echo $(( (a << 24) | (b << 16) | (c << 8) | d )) +} + +in_subnet() { + local ipn netn len mask + [ -n "${2:-}" ] || return 1 + ipn=$(ip2int "$1") || return 1 + netn=$(ip2int "${2%/*}") || return 1 + len=${2#*/} + [[ "$len" =~ ^[0-9]+$ ]] && [ "$len" -le 32 ] || return 1 + mask=$(( len == 0 ? 0 : (0xFFFFFFFF << (32 - len)) & 0xFFFFFFFF )) + [ $(( ipn & mask )) -eq $(( netn & mask )) ] +} + +# Admin SSH clients: PXE_SSH_CLIENT, our own SSH_CONNECTION, and every +# established peer of sshd (port 22). IPv4-mapped IPv6 is unwrapped. +ssh_peers() { + { + [ -n "${PXE_SSH_CLIENT:-}" ] && echo "$PXE_SSH_CLIENT" + [ -n "${SSH_CONNECTION:-}" ] && echo "${SSH_CONNECTION%% *}" + ss -Htn state established '( sport = :22 )' 2>/dev/null | awk '{print $NF}' | + sed -E 's/:[0-9]+$//; s/^\[//; s/\]$//' + } | sed -E 's/^::ffff://' | grep -E '^[0-9a-fA-F.:]+$' | sort -u +} + +# --- pre-flight -------------------------------------------------------------- + +# 0 = safe to apply. Prints every reason it is not. +preflight() { + local pif cif peer dev bad=0 + pif=$(iface_of "$PXE_MAC") + cif=$(iface_of "$CORP_MAC") + if [ -z "$pif" ]; then + log "REFUSED: no physical NIC has pxe_nic_mac $PXE_MAC" + return 1 + fi + if [ -z "$cif" ]; then + log "REFUSED: no physical NIC has corp_nic_mac $CORP_MAC" + return 1 + fi + if default_route_devs | grep -qxF "$pif"; then + log "REFUSED: $pif (pxe_nic_mac $PXE_MAC) holds the default route" + bad=1 + fi + if has_dhcp_lease "$pif"; then + log "REFUSED: $pif (pxe_nic_mac $PXE_MAC) has a DHCP lease" + bad=1 + fi + while IFS= read -r peer; do + [ -n "$peer" ] || continue + dev=$(route_dev "$peer") + [ "$dev" = "$pif" ] || continue + if in_subnet "$peer" "${PXE_SUBNET:-}"; then + continue # an admin on the imaging LAN keeps 172.16.9.1 + fi + log "REFUSED: the SSH client $peer is reached through $pif (pxe_nic_mac $PXE_MAC)" + bad=1 + done < <(ssh_peers) + if [ "$bad" -ne 0 ]; then + log "The NIC named by pxe_nic_mac looks like the GE corporate one: pxe_nic_mac" + log "and corp_nic_mac are probably swapped, or the cables are. It would become" + log "static ${PXE_SUBNET:-imaging} with no gateway and cut this box off. Nothing was changed." + log "NICs now: $(for d in "$SYSFS_NET"/*; do [ -e "$d/device" ] && printf '%s=%s ' "$(basename "$d")" "$(cat "$d/address" 2>/dev/null)"; done)" + return 1 + fi + return 0 +} + +# --- renaming ---------------------------------------------------------------- + +link_rename() { + local cur=$1 new=$2 + log "renaming $cur to $new" + if ip link set dev "$cur" down && + ip link set dev "$cur" name "$new" && + ip link set dev "$new" up; then + return 0 + fi + log "ERROR: rename $cur -> $new failed" + ip link set dev "$cur" up 2>/dev/null + ip link set dev "$new" up 2>/dev/null + return 1 +} + +# set_names MAC1 NAME1 MAC2 NAME2: give each MAC its name. A NIC that must +# change goes through a temporary name first, so a swap (pxe0 <-> corp0) +# works in both directions. Temporary names carry this run's PID, so they +# never collide with a name left over from an earlier (failed) pass. +set_names() { + local -a macs=("$1" "$3") want=("$2" "$4") cur=() + local i + RENAME_SEQ=$(( ${RENAME_SEQ:-0} + 1 )) + for i in 0 1; do + [ -n "${want[i]}" ] || return 0 + cur[i]=$(iface_of "${macs[i]}") + if [ -z "${cur[i]}" ]; then + log "ERROR: no NIC with MAC ${macs[i]}" + return 1 + fi + done + for i in 0 1; do + [ "${cur[i]}" = "${want[i]}" ] && continue + link_rename "${cur[i]}" "pxt$$-$RENAME_SEQ$i" || return 1 + cur[i]=pxt$$-$RENAME_SEQ$i + done + for i in 0 1; do + [ "${cur[i]}" = "${want[i]}" ] && continue + link_rename "${cur[i]}" "${want[i]}" || return 1 + done + for i in 0 1; do + if [ "$(iface_of "${macs[i]}")" != "${want[i]}" ]; then + log "ERROR: MAC ${macs[i]} is not named ${want[i]} after renaming" + return 1 + fi + done + return 0 +} + names_ok() { [ "$(iface_of "$PXE_MAC")" = "$PXE_NAME" ] && [ "$(iface_of "$CORP_MAC")" = "$CORP_NAME" ] } +# --- backup / restore -------------------------------------------------------- + # Other netplan files that configure wired NICs (installer leftovers, an # en* DHCP match): they would put a second config on the two NICs. Files # without "ethernets" (WiFi, NetworkManager wifi profiles) are left alone. @@ -61,6 +230,94 @@ other_files() { ! -path "$TARGET" -exec grep -l 'ethernets' {} + 2>/dev/null | sort } +# State of one apply, next to (not inside) the backup, so restoring the +# backup never copies it into /etc/netplan. +write_state() { + { + echo "NETPLAN_DIR=$NETPLAN_DIR" + echo "PXE_MAC=$PXE_MAC" + echo "ORIG_PXE_IF=$ORIG_PXE_IF" + echo "CORP_MAC=$CORP_MAC" + echo "ORIG_CORP_IF=$ORIG_CORP_IF" + } >"$BACKUP.state" +} + +read_state() { + local k v + while IFS='=' read -r k v; do + case "$k" in + NETPLAN_DIR) [[ "$v" =~ ^/[A-Za-z0-9/._-]*$ ]] && NETPLAN_DIR=$v ;; + PXE_MAC|CORP_MAC) [[ "$v" =~ ^([0-9a-f]{2}:){5}[0-9a-f]{2}$ ]] && printf -v "$k" '%s' "$v" ;; + ORIG_PXE_IF|ORIG_CORP_IF) [[ "$v" =~ ^[A-Za-z0-9._-]{0,15}$ ]] && printf -v "$k" '%s' "$v" ;; + esac + done <"$BACKUP.state" + TARGET=$NETPLAN_DIR/50-cloud-init.yaml +} + +cancel_revert() { + [ -n "${REVERT_UNIT:-}" ] || return 0 + systemctl stop "$REVERT_UNIT.timer" "$REVERT_UNIT.service" >/dev/null 2>&1 + if systemctl is-active --quiet "$REVERT_UNIT.timer" 2>/dev/null; then + log "WARNING: could not cancel $REVERT_UNIT.timer (the committed marker still stops it)" + fi +} + +# Put the backed-up config AND the previous NIC names back. +restore() { + local rc=0 + log "restoring the previous config from $BACKUP" + find "$NETPLAN_DIR" -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.yml' \) -delete + cp -a "$BACKUP/." "$NETPLAN_DIR/" + if [ -n "${ORIG_PXE_IF:-}" ] && [ -n "${ORIG_CORP_IF:-}" ]; then + set_names "$PXE_MAC" "$ORIG_PXE_IF" "$CORP_MAC" "$ORIG_CORP_IF" || + { log "ERROR: could not put the previous NIC names back"; rc=1; } + fi + netplan generate && netplan apply || { log "ERROR: netplan apply of the restored config failed"; rc=1; } + touch "$BACKUP.reverted" + return $rc +} + +fail() { + log "ERROR: $*" + restore + cancel_revert + log "the previous config is back in place; nothing applied" + exit 1 +} + +# --- modes ------------------------------------------------------------------- + +MODE=apply +case "${1:-}" in + --preflight) MODE=preflight; shift ;; + --revert) + [ $# -eq 2 ] || { echo "usage: $0 --revert BACKUP_DIR" >&2; exit 2; } + BACKUP=${2%/} + [ -d "$BACKUP" ] && [ -f "$BACKUP.state" ] || { log "revert: no backup/state at $BACKUP"; exit 1; } + exec 9>"$LOCK_FILE" + flock -w 600 9 || log "revert: lock still held after 600 s; reverting anyway" + if [ -e "$BACKUP.committed" ] || [ -e "$BACKUP.reverted" ]; then + log "revert: $BACKUP already committed or reverted; nothing to do" + exit 0 + fi + PXE_MAC="" CORP_MAC="" ORIG_PXE_IF="" ORIG_CORP_IF="" + read_state + log "revert: the apply was not confirmed within its window" + restore + exit $? + ;; +esac + +if [ $# -ne 5 ]; then + echo "usage: $0 [--preflight] STAGED_YAML PXE_NAME PXE_MAC CORP_NAME CORP_MAC" >&2 + exit 2 +fi +STAGED=$1 +PXE_NAME=$2 +PXE_MAC=${3,,} +CORP_NAME=$4 +CORP_MAC=${5,,} + # 1. validate in a scratch root SCRATCH=$(mktemp -d) trap 'rm -rf "$SCRATCH"' EXIT @@ -78,72 +335,98 @@ if cmp -s "$STAGED" "$TARGET" && [ -z "$(other_files)" ] && names_ok; then exit 0 fi -# 3. back up -BACKUP=/var/backups/netplan-$(date +%Y%m%d-%H%M%S) +# 3. pre-flight +if ! preflight; then + exit 1 +fi +if [ "$MODE" = preflight ]; then + log "pre-flight passed; the netplan config would change" + echo NEEDED + exit 0 +fi + +exec 9>"$LOCK_FILE" +flock -w 60 9 || { log "ERROR: another pxe-netplan-apply holds $LOCK_FILE; nothing changed"; exit 1; } + +# 4. back up, record names, arm the auto-revert +ORIG_PXE_IF=$(iface_of "$PXE_MAC") +ORIG_CORP_IF=$(iface_of "$CORP_MAC") +CORP_HAD_V4=0 +has_v4 "$ORIG_CORP_IF" && CORP_HAD_V4=1 +BACKUP=$BACKUP_ROOT/netplan-$(date +%Y%m%d-%H%M%S) mkdir -p "$BACKUP" cp -a "$NETPLAN_DIR/." "$BACKUP/" -log "previous config saved in $BACKUP" +write_state +log "previous config saved in $BACKUP (NICs: $ORIG_PXE_IF, $ORIG_CORP_IF)" -restore() { - log "restoring the previous config from $BACKUP" - find "$NETPLAN_DIR" -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.yml' \) -delete - cp -a "$BACKUP/." "$NETPLAN_DIR/" - netplan generate && netplan apply -} +REVERT_UNIT="" +if [ "$REVERT_AFTER" -gt 0 ] 2>/dev/null && command -v systemd-run >/dev/null; then + REVERT_UNIT=pxe-netplan-revert-$(basename "$BACKUP" | tr -cd 'A-Za-z0-9-') + if systemd-run --quiet --unit="$REVERT_UNIT" --on-active="$REVERT_AFTER" \ + --timer-property=AccuracySec=1s "$SELF" --revert "$BACKUP"; then + log "auto-revert armed: $REVERT_UNIT.timer restores $BACKUP in ${REVERT_AFTER}s unless this run confirms" + else + log "WARNING: could not arm the auto-revert timer; relying on the in-script checks" + REVERT_UNIT="" + fi +else + log "WARNING: auto-revert off (no systemd-run or PXE_NETPLAN_REVERT_AFTER=0)" +fi -# 4. install, moving aside the other wired configs (an en* DHCP match would +# An SSH drop (HUP) must not stop us half-way; TERM/INT restore. +trap '' HUP +trap 'fail "interrupted"' INT TERM + +# 5. install, moving aside the other wired configs (an en* DHCP match would # put a DHCP client on the imaging NIC too) -other_files | while IFS= read -r f; do +while IFS= read -r f; do + [ -n "$f" ] || continue log "moving aside $f (copy in $BACKUP)" rm -f "$f" -done +done < <(other_files) install -m 600 "$STAGED" "$TARGET" -if ! netplan generate; then - log "ERROR: netplan generate failed" - restore - exit 1 -fi +netplan generate || fail "netplan generate failed" -# 5. stable names now, so dnsmasq/UFW/Samba (which use them) work before +# 6. stable names now, so dnsmasq/UFW/Samba (which use them) work before # the next reboot. Does not depend on the netplan version renaming. -rename() { - local mac=$1 want=$2 cur - cur=$(iface_of "$mac") - if [ -z "$cur" ]; then - log "WARNING: no NIC with MAC $mac" - return 0 - fi - [ "$cur" = "$want" ] && return 0 - log "renaming $cur ($mac) to $want" - if ! { ip link set dev "$cur" down && - ip link set dev "$cur" name "$want" && - ip link set dev "$want" up; }; then - log "WARNING: rename $cur -> $want failed; it applies at the next reboot" - ip link set dev "$cur" up 2>/dev/null - fi - return 0 -} -rename "$PXE_MAC" "$PXE_NAME" -rename "$CORP_MAC" "$CORP_NAME" +set_names "$PXE_MAC" "$PXE_NAME" "$CORP_MAC" "$CORP_NAME" || fail "NIC rename failed" -if ! netplan apply; then - log "ERROR: netplan apply failed" - restore - exit 1 +# 7. apply +netplan apply || fail "netplan apply failed" + +# 8. check +names_ok || fail "$PXE_NAME / $CORP_NAME are not on MACs $PXE_MAC / $CORP_MAC after apply" +ok=0 +for _ in $(seq 1 "$PXE_WAIT"); do + if has_v4 "$PXE_NAME"; then ok=1; break; fi + sleep 1 +done +[ "$ok" -eq 1 ] || fail "$PXE_NAME has no IPv4 address after ${PXE_WAIT}s" +if [ "$CORP_HAD_V4" -eq 1 ]; then + ok=0 + for _ in $(seq 1 "$CORP_WAIT"); do + if has_v4 "$CORP_NAME"; then ok=1; break; fi + sleep 1 + done + [ "$ok" -eq 1 ] || fail "$CORP_NAME had IPv4 before and has none after ${CORP_WAIT}s" +elif ! has_v4 "$CORP_NAME"; then + log "WARNING: $CORP_NAME has no IPv4 address (it had none before either: corp cable unplugged or no DHCP)" fi +while IFS= read -r peer; do + [ -n "$peer" ] || continue + dev=$(route_dev "$peer") + if [ "$dev" = "$PXE_NAME" ] && ! in_subnet "$peer" "${PXE_SUBNET:-}"; then + fail "the SSH client $peer now routes through $PXE_NAME" + fi + if [ -z "$dev" ]; then + fail "no route to the SSH client $peer after apply" + fi +done < <(ssh_peers) -# 6. check -for _ in $(seq 1 20); do - ip -4 -br addr show dev "$PXE_NAME" 2>/dev/null | grep -q ' [0-9]' && break - sleep 1 -done -ip -br addr show dev "$PXE_NAME" 2>/dev/null || - log "WARNING: $PXE_NAME not present yet (reboot to apply the .link names)" -for _ in $(seq 1 30); do - ip -4 -br addr show dev "$CORP_NAME" 2>/dev/null | grep -q ' [0-9]' && break - sleep 1 -done -ip -4 -br addr show dev "$CORP_NAME" 2>/dev/null | grep -q ' [0-9]' || - log "WARNING: $CORP_NAME has no IPv4 address yet (corp cable unplugged or DHCP slow)" +touch "$BACKUP.committed" +cancel_revert +trap - INT TERM +ip -br addr show dev "$PXE_NAME" 2>/dev/null +ip -br addr show dev "$CORP_NAME" 2>/dev/null echo CHANGED exit 0 diff --git a/playbook/pxe_server_setup.yml b/playbook/pxe_server_setup.yml index 57724a8..0a9de42 100644 --- a/playbook/pxe_server_setup.yml +++ b/playbook/pxe_server_setup.yml @@ -30,6 +30,7 @@ - ansible - wimtools - conntrack + - networkd-dispatcher # carrier hook (pxe-nic-carrier) - sbsigntool # sbverify: iPXE update signature checks - p7zip-full # 7z: Dell driver packs, catalog cabs, Blancco ISO - zip # driver zips PESetup unpacks @@ -62,6 +63,9 @@ ipxe_shim_sha256: "5eecca2780bd49c900565e124516a1bd666ec5e012825f34991b6ba1ef2fa6cf" # ipxe/shim ipxe-16.1 ipxe-shimx64.efi ipxe_snponly_sha256: "b1e67c3e4a1e8708ddfd0079ad4505e3a02245acb55ee9a95437ab3c507be82a" # x86_64-sb/snponly.efi ipxe_bios_sha256: "0fb21c695c82fd70d4f89732ddcd285063776e2bbd71de9a2c39e9cfe4b180b8" # x86_64/ipxe.pxe + # Release the pins above come from. A re-run does not replace files the + # webapp's Updates page installed from a NEWER release (installed.json). + ipxe_release_tag: "v2.0.0" # Zscaler Root CA: GE's Zscaler re-signs GitHub TLS, so the iPXE update # check fails without it. Repo copy first, else the copy on the image share. zscaler_ca_src: "{{ playbook_dir }}/certs/zscaler-root-ca.crt" @@ -126,6 +130,37 @@ ufw_corp_rules: - { port: "22", proto: tcp } - { port: "9009", proto: tcp } + # Who may reach SSH and the webapp (9009, no login, runs as root) on the + # GE corporate NIC. Empty (the default) = all of GE, as before, with a + # loud warning every run. Set to the admin subnets/hosts, e.g. + # ["10.134.48.0/24", "3.12.34.56/32"]: UFW then allows 22 and 9009 on + # the corp NIC from these only. The play refuses to restrict if your own + # SSH session is not inside one of them. See docs/SITE-CUSTOMIZATION.md. + corp_admin_cidrs: [] + # false: sshd refuses password logins except from the imaging subnet + # and localhost (keys only from the GE side). Default true keeps the + # shared 'pxe' password working from GE, so nobody is locked out; the + # play refuses false while the pxe user has no authorized_keys. + corp_ssh_password_auth: true + # Re-run safety: when imaging is in progress (SMB sessions open on 445 + # or DHCP leases granted in the last pxe_busy_window_min minutes) a + # re-run does not bounce the NICs or restart dnsmasq/Apache/Samba. + # A needed network change then stops the play with a message; + # -e allow_disruptive=true overrides. + allow_disruptive: false + pxe_busy_window_min: 30 + # networkd-dispatcher states the carrier hook (pxe-nic-carrier) runs in. + carrier_hook_states: + - carrier.d + - degraded.d + - degraded-carrier.d + - enslaved.d + - routable.d + - no-carrier.d + - off.d + # The admin's SSH client (empty when not run over SSH, or when sudo + # dropped SSH_CONNECTION; the checks also read sshd's live peers). + pxe_ssh_client: "{{ (lookup('env', 'SSH_CONNECTION') | default('', true)).split(' ')[0] }}" # --- Blancco UEFI / Secure Boot path --- # Blancco's UEFI entries boot a Canonical-signed Ubuntu HWE kernel # (through Ubuntu's shim when Secure Boot is on), then switch_root into @@ -136,6 +171,13 @@ # major+minor ("7.0.0-34-generic" -> "70"), as GetPxeScript.aspx expects: # vmlinuz-ubuntu-70, kexec-initrd-70.img; modules are # kmod-.tar.gz (blancco-init.sh fetches kmod-$(uname -r)). + # The pin lives in three places; this var is the source: + # 1. here (blancco_kver) + # 2. scripts/download-packages.sh reads its BLANCCO_KVER default from + # this line (packages linux-image/modules-) + # 3. playbook/ipxe/GetPxeScript.aspx names the files by suffix + # (vmlinuz-ubuntu-70, kexec-initrd-70.img); the play fails if that + # suffix is not blancco_ksuffix. blancco_kver: "7.0.0-34-generic" blancco_ksuffix: "{{ blancco_kver.split('.')[0] }}{{ blancco_kver.split('.')[1] }}" image_types: @@ -208,6 +250,145 @@ Fix playbook/site-vars.yml and re-run. quiet: yes + # --- Re-run safety (nothing below changes the box until these pass) --- + - name: "Re-run safety: is imaging in progress? (SMB sessions, recent DHCP leases)" + args: + executable: /bin/bash + shell: | + smb=$(ss -Htn state established '( sport = :445 )' 2>/dev/null | wc -l) + recent=0 + L=/var/lib/misc/dnsmasq.leases + lease="{{ pxe_dhcp_lease }}" + case "$lease" in + *[0-9]d) secs=$(( ${lease%d} * 86400 )) ;; + *[0-9]h) secs=$(( ${lease%h} * 3600 )) ;; + *[0-9]m) secs=$(( ${lease%m} * 60 )) ;; + *[0-9]s) secs=${lease%s} ;; + *[0-9]) secs=$lease ;; + *) secs=0 ;; + esac + if [ -f "$L" ] && [ "$secs" -gt 0 ]; then + # Field 1 is the expiry; granted = expiry - lease time. + recent=$(awk -v now="$(date +%s)" -v lease="$secs" -v win="{{ (pxe_busy_window_min | int) * 60 }}" \ + '$1 > 0 && ($1 - lease) > (now - win) {n++} END {print n + 0}' "$L") + fi + echo "$smb $recent" + register: pxe_busy + changed_when: false + + - name: "Re-run safety: decide" + set_fact: + pxe_busy_smb: "{{ pxe_busy.stdout.split()[0] | int }}" + pxe_busy_leases: "{{ pxe_busy.stdout.split()[1] | int }}" + pxe_hold_disruptive: "{{ ((pxe_busy.stdout.split()[0] | int) > 0 or (pxe_busy.stdout.split()[1] | int) > 0) and not (allow_disruptive | bool) }}" + + - name: "Re-run safety: imaging in progress, disruptive steps are held" + debug: + msg: >- + {{ pxe_busy_smb }} SMB session(s) open and {{ pxe_busy_leases }} + DHCP lease(s) granted in the last {{ pxe_busy_window_min }} min: + imaging looks active. This run will NOT bounce the NICs or restart + dnsmasq/Apache/Samba (a needed network change stops the play). + Re-run when idle, or with -e allow_disruptive=true. + when: pxe_hold_disruptive | bool + + - name: "Corp NIC: corp_admin_cidrs must be valid and include your own SSH session" + # Restricting SSH on the GE NIC to subnets that do not contain the + # admin running this play would lock them out on the next login. + when: corp_admin_cidrs | length > 0 + environment: + PXE_SSH_CLIENT: "{{ pxe_ssh_client }}" + CIDRS: "{{ corp_admin_cidrs | join(' ') }}" + PXE_SUBNET: "{{ pxe_subnet }}" + args: + executable: /bin/bash + shell: | + peers=$( { echo "${PXE_SSH_CLIENT:-}"; ss -Htn state established '( sport = :22 )' 2>/dev/null | awk '{print $NF}'; } ) + python3 - "$peers" <<'PY' + import ipaddress, os, re, sys + try: + nets = [ipaddress.ip_network(c, strict=False) for c in os.environ["CIDRS"].split()] + except ValueError as ex: + sys.exit(f"REFUSED: corp_admin_cidrs has an invalid entry: {ex}") + local = [ipaddress.ip_network(os.environ["PXE_SUBNET"], strict=False), + ipaddress.ip_network("127.0.0.0/8"), ipaddress.ip_network("::1/128")] + bad = [] + def parse(raw): + for cand in (raw, re.sub(r":\d+$", "", raw)): + cand = cand.strip("[]") + cand = cand[7:] if cand.startswith("::ffff:") else cand + try: + return ipaddress.ip_address(cand) + except ValueError: + pass + return None + for raw in sys.argv[1].split(): + ip = parse(raw) + if ip is None: + continue + if any(ip in n for n in local) or any(ip in n for n in nets): + continue + bad.append(str(ip)) + if bad: + sys.exit("REFUSED: SSH client(s) %s are not inside corp_admin_cidrs %s; restricting " + "would lock them out. Add their subnet, or run from inside one. Nothing changed." + % (", ".join(sorted(set(bad))), os.environ["CIDRS"])) + print("corp_admin_cidrs cover every current SSH client") + PY + changed_when: false + + - name: "SSH: keys required before password logins are turned off on the GE side" + when: not (corp_ssh_password_auth | bool) + shell: | + home=$(getent passwd pxe | cut -d: -f6) + if [ -n "$home" ] && [ -s "$home/.ssh/authorized_keys" ] && + grep -qE '^(ssh-|ecdsa-|sk-)' "$home/.ssh/authorized_keys"; then + echo "pxe has $(grep -cE '^(ssh-|ecdsa-|sk-)' "$home/.ssh/authorized_keys") key(s)" + else + echo "REFUSED: corp_ssh_password_auth is false but pxe has no ~/.ssh/authorized_keys;" >&2 + echo "the GE side would be locked out. Add a key (ssh-copy-id pxe@) first." >&2 + exit 1 + fi + changed_when: false + + - name: "NICs: render the netplan config for the pre-flight check (in /run, not installed)" + template: + src: templates/netplan-pxe.yaml.j2 + dest: /run/pxe-server-netplan.check.yaml + mode: '0600' + changed_when: false + + - name: "NICs: pre-flight (refuses swapped MACs / mis-cabled NICs before anything changes)" + # Refuses when the NIC pxe_nic_mac names holds the default route, a + # DHCP lease or the route to an SSH client: it would become static + # with no gateway and cut the box off. Runs from the USB copy, so + # nothing on the box has been touched yet. + command: >- + bash {{ usb_mount }}/pxe-server-helpers/pxe-netplan-apply.sh --preflight + /run/pxe-server-netplan.check.yaml + {{ pxe_nic_name }} {{ pxe_nic_mac | lower }} + {{ corp_nic_name }} {{ corp_nic_mac | lower }} + environment: + PXE_SUBNET: "{{ pxe_subnet }}" + PXE_SSH_CLIENT: "{{ pxe_ssh_client }}" + register: netplan_preflight + changed_when: false + + - name: "NICs: pre-flight result" + debug: + msg: "{{ netplan_preflight.stdout_lines }}" + + - name: "NICs: a network change is needed but imaging is in progress" + fail: + msg: >- + The netplan config / NIC names must change, which bounces the NICs, + but imaging is active ({{ pxe_busy_smb }} SMB sessions, + {{ pxe_busy_leases }} recent leases). Nothing was changed. Re-run + when idle, or with -e allow_disruptive=true. + when: + - pxe_hold_disruptive | bool + - netplan_preflight.stdout_lines | last | default('') == 'NEEDED' + - name: "NICs: stop cloud-init from rewriting the network config" copy: dest: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg @@ -234,6 +415,9 @@ /usr/local/sbin/pxe-netplan-apply.sh /etc/pxe-server-netplan.yaml {{ pxe_nic_name }} {{ pxe_nic_mac | lower }} {{ corp_nic_name }} {{ corp_nic_mac | lower }} + environment: + PXE_SUBNET: "{{ pxe_subnet }}" + PXE_SSH_CLIENT: "{{ pxe_ssh_client }}" register: netplan_apply changed_when: netplan_apply.stdout_lines | last | default('') == 'CHANGED' @@ -252,9 +436,12 @@ net.ipv6.conf.all.forwarding = 0 net.ipv6.conf.default.forwarding = 0 # Apache listens on {{ pxe_server_ip }} only (ports 80/4433). Let it - # bind before networkd has put the address on {{ pxe_nic_name }} (no - # carrier at boot, or apache starting first), or apache2 fails to - # start and takes the webapp on 9009 down with it. + # bind before networkd has put the address on {{ pxe_nic_name }}, or + # apache2 fails to start and takes the webapp on 9009 down with it. + # The netplan config sets ignore-carrier on {{ pxe_nic_name }}, so a + # missing link no longer keeps the address off; this stays for the + # boot-order race (apache2 is ordered only after network.target, + # not after networkd has configured the address). net.ipv4.ip_nonlocal_bind = 1 register: pxe_net_sysctl @@ -416,6 +603,62 @@ - pxe-nic-check.service - pxe-nic-check.timer + # Carrier hook: networkd-dispatcher runs pxe-nic-carrier on every link + # state change; it starts pxe-nic-carrier.service, which stops dnsmasq + # when the imaging link drops and starts it again (through the + # ExecStartPre guard) when it comes back. pxe-nic-check, its service + # and pxe-dhcp-hook.sh are (re)copied by the tasks above/below. + - name: "Carrier hook: networkd-dispatcher installed (offline repo)" + apt: + name: networkd-dispatcher + state: present + + - name: "Carrier hook: deploy pxe-nic-carrier" + copy: + src: "{{ usb_mount }}/pxe-server-helpers/pxe-nic-carrier" + dest: /usr/local/sbin/pxe-nic-carrier + owner: root + group: root + mode: '0755' + + - name: "Carrier hook: install pxe-nic-carrier.service (started by the hook, not enabled)" + copy: + src: "{{ usb_mount }}/pxe-server-helpers/pxe-nic-carrier.service" + dest: /etc/systemd/system/pxe-nic-carrier.service + mode: '0644' + register: carrier_unit + + - name: "Carrier hook: systemd daemon-reload" + systemd: + daemon_reload: yes + when: carrier_unit is changed + + - name: "Carrier hook: networkd-dispatcher state directories" + file: + path: "/etc/networkd-dispatcher/{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: "{{ carrier_hook_states }}" + notify: restart networkd-dispatcher + + - name: "Carrier hook: link the hook into each state" + file: + src: /usr/local/sbin/pxe-nic-carrier + dest: "/etc/networkd-dispatcher/{{ item }}/50-pxe-nic-carrier" + state: link + owner: root + group: root + loop: "{{ carrier_hook_states }}" + notify: restart networkd-dispatcher + + - name: "Carrier hook: networkd-dispatcher enabled and running" + systemd: + name: networkd-dispatcher + enabled: yes + state: started + - name: "NIC guard: enable watch timer" systemd: name: pxe-nic-check.timer @@ -458,6 +701,47 @@ - blancco - memtest + - name: "Menu: the Blancco file names in the menu must match blancco_kver" + # GetPxeScript.aspx names the UEFI Blancco files by suffix + # (vmlinuz-ubuntu-70, kexec-initrd-70.img); the play builds them from + # blancco_kver. A mismatch boots a file that is never built. + shell: | + M="{{ usb_mount }}/ipxe/GetPxeScript.aspx" + found=$(grep -oE '(vmlinuz-ubuntu|kexec-initrd)-[0-9]+' "$M" | sed 's/.*-//' | sort -u | tr '\n' ' ') + echo "menu suffixes: ${found:-none}" + for x in $found; do + if [ "$x" != "{{ blancco_ksuffix }}" ]; then + echo "REFUSED: $M references Blancco files with suffix $x but blancco_kver" >&2 + echo "{{ blancco_kver }} builds suffix {{ blancco_ksuffix }}. Change them together" >&2 + echo "(playbook var blancco_kver, scripts/download-packages.sh, the menu)." >&2 + exit 1 + fi + done + register: menu_ksuffix + changed_when: false + + - name: "Menu: does the live menu carry a staged Blancco test entry?" + # The Blancco Releases page (webapp/services/blancco_update.py) adds a + # test entry between these markers; overwriting the menu would drop it + # while the page still shows it as added. + shell: | + F="{{ web_root }}/Altiris/iPXE/GetPxeScript.aspx" + [ -f "$F" ] || exit 0 + if grep -qF '# BEGIN blancco-staged test entry' "$F" || grep -qE '^item[[:space:]]+blancco_staged[[:space:]]' "$F"; then + echo STAGED-TEST + fi + register: menu_test_entry + changed_when: false + + - name: "Menu: NOT replacing the live menu (Blancco test entry present)" + debug: + msg: >- + WARNING: {{ web_root }}/Altiris/iPXE/GetPxeScript.aspx carries the + staged Blancco test entry (Updates > Blancco). The menu was NOT + updated from {{ usb_mount }}/ipxe/GetPxeScript.aspx. Remove the + test entry on that page (or promote the release), then re-run. + when: "'STAGED-TEST' in menu_test_entry.stdout" + - name: "Install GetPxeScript.aspx (iPXE boot menu)" # The live menu, tracked as a file (playbook/ipxe/GetPxeScript.aspx). # WinPE skips the Secure Boot warning when iPXE reads Secure Boot as on @@ -470,6 +754,7 @@ dest: "{{ web_root }}/Altiris/iPXE/GetPxeScript.aspx" mode: '0644' backup: yes + when: "'STAGED-TEST' not in menu_test_entry.stdout" # Boot files (80) and the iPXE menu (4433) are served on the imaging IP # only, never on the GE NIC. 9009 (webapp) stays on all addresses: admins @@ -700,6 +985,20 @@ group: pxe mode: '0775' + # Imaging stage reporter: the bays run it from the shared Applications + # tree and post their stage to the webapp's /imaging dashboard. + - name: "Imaging status: create _shared/Applications/extra/pxe-status" + file: + path: "{{ samba_share }}/_shared/Applications/extra/pxe-status" + state: directory + mode: '0755' + + - name: "Imaging status: deploy Send-ImagingStage.ps1" + copy: + src: "{{ usb_mount }}/imaging-status/Send-ImagingStage.ps1" + dest: "{{ samba_share }}/_shared/Applications/extra/pxe-status/Send-ImagingStage.ps1" + mode: '0644' + - name: "Deploy BIOS check script + manifest to winpeapps/_shared/BIOS/" # Path matches what startnet.cmd reads at WinPE boot: # net use B: \\172.16.9.1\winpeapps\_shared @@ -916,6 +1215,27 @@ force: no loop: "{{ image_types }}" + - name: "Unattend: lint the answer files before they replace the live ones" + # A bad answer file stops every bay imaged with it at "Windows could + # not parse or process unattend answer file". The copies below use + # force: yes, so this gate is the only check. scripts/lint-unattend.py + # rides on the USB as scripts/ next to playbook/ (build-usb.sh), and + # is in the repo at the same relative path. + args: + executable: /bin/bash + shell: | + for L in "{{ usb_mount }}/../scripts/lint-unattend.py" "{{ playbook_dir }}/../scripts/lint-unattend.py"; do + if [ -f "$L" ]; then + exec python3 "$L" --local \ + "{{ usb_mount }}/FlatUnattendW10-shopfloor.xml" \ + "{{ usb_mount }}/FlatUnattendW10.xml" \ + "{{ usb_mount }}/FlatUnattendW10-engineer.xml" + fi + done + echo "ERROR: scripts/lint-unattend.py not found next to {{ usb_mount }} or {{ playbook_dir }}" >&2 + exit 1 + changed_when: false + - name: "Deploy shopfloor unattend.xml template" copy: src: "{{ usb_mount }}/FlatUnattendW10-shopfloor.xml" @@ -1006,12 +1326,49 @@ check "$W/ipxeboot/x86_64-sb/snponly.efi" "{{ ipxe_snponly_sha256 }}" check "$W/ipxeboot/x86_64/ipxe.pxe" "{{ ipxe_bios_sha256 }}" put() { install -m 0644 -o root -g root "$1" "$T/.$2.new" && mv -f "$T/.$2.new" "$T/$2"; } - put "$U/ipxe-shimx64.efi" ipxe-shim.efi - put "$W/ipxeboot/x86_64-sb/snponly.efi" ipxe.efi - put "$W/ipxeboot/x86_64/ipxe.pxe" ipxe.pxe + # Never downgrade what the webapp's Updates page installed from a + # newer release than ipxe_release_tag: a file whose live sha256 still + # matches its installed.json record with a newer tag is kept. The + # shim and ipxe.efi go together (the shim must trust ipxe.efi's CA). + AHEAD=$(python3 - /var/lib/pxe-webapp/updates/ipxe/installed.json "$T" "{{ ipxe_release_tag }}" <<'PY' + import hashlib, json, os, re, sys + log, tftp, pin = sys.argv[1:4] + def ver(t): + m = re.search(r"(\d+)\.(\d+)(?:\.(\d+))?", t or "") + return tuple(int(x or 0) for x in m.groups()) if m else None + try: + data = json.load(open(log)) + except (OSError, ValueError): + data = {} + data = data if isinstance(data, dict) else {} + for name in ("ipxe-shim.efi", "ipxe.efi", "ipxe.pxe"): + rec, path = data.get(name), os.path.join(tftp, name) + if not isinstance(rec, dict) or not os.path.isfile(path): + continue + if hashlib.sha256(open(path, "rb").read()).hexdigest() != rec.get("sha256"): + continue + a, b = ver(rec.get("tag")), ver(pin) + if a and b and a > b: + print(f"{name}={rec.get('tag')}") + PY + ) + done="" + if grep -qE '^(ipxe-shim\.efi|ipxe\.efi)=' <<<"$AHEAD"; then + echo "KEPT ipxe-shim.efi + ipxe.efi: the Updates page installed a newer release ($(grep -E '^ipxe(-shim)?\.efi=' <<<"$AHEAD" | tr '\n' ' ')) than the playbook's {{ ipxe_release_tag }}" + else + put "$U/ipxe-shimx64.efi" ipxe-shim.efi + put "$W/ipxeboot/x86_64-sb/snponly.efi" ipxe.efi + done="ipxe-shim.efi, ipxe.efi (snponly)" + fi + if grep -qE '^ipxe\.pxe=' <<<"$AHEAD"; then + echo "KEPT ipxe.pxe: the Updates page installed $(grep -E '^ipxe\.pxe=' <<<"$AHEAD"), newer than {{ ipxe_release_tag }}" + else + put "$W/ipxeboot/x86_64/ipxe.pxe" ipxe.pxe + done="${done:+$done, }ipxe.pxe" + fi # Rollback boot file: the old Broadcom-signed build (boot-files/ipxe.efi). - if [ -f "$U/ipxe.efi" ]; then put "$U/ipxe.efi" ipxe-bcm.efi; fi - echo "installed ipxe-shim.efi, ipxe.efi (snponly), ipxe.pxe$( [ -f "$U/ipxe.efi" ] && echo ', ipxe-bcm.efi')" + if [ -f "$U/ipxe.efi" ]; then put "$U/ipxe.efi" ipxe-bcm.efi; done="${done:+$done, }ipxe-bcm.efi"; fi + echo "installed: ${done:-nothing}" args: executable: /bin/bash register: ipxe_chain @@ -1049,17 +1406,42 @@ set_fact: blancco_kernel_ok: "{{ blancco_kernel_files.results | map(attribute='stat.exists') | min }}" - - name: "Blancco: warn when the Blancco kernel is missing" + - name: "Blancco: are the UEFI Blancco files already served?" + stat: + path: "{{ web_root }}/blancco/{{ item }}" + loop: + - "vmlinuz-ubuntu-{{ blancco_ksuffix }}" + - "kexec-initrd-{{ blancco_ksuffix }}.img" + register: blancco_served + when: not blancco_kernel_ok + + # The menu boots these files (checked above: same suffix). Without the + # kernel they cannot be built; with no earlier build in place, every + # UEFI Blancco boot would fail, so that is fatal, not a warning. + - name: "Blancco: the kernel is missing and the menu's files were never built" + fail: + msg: >- + /boot/vmlinuz-{{ blancco_kver }} or /lib/modules/{{ blancco_kver }} + is missing, and {{ web_root }}/blancco has no + vmlinuz-ubuntu-{{ blancco_ksuffix }} / + kexec-initrd-{{ blancco_ksuffix }}.img from an earlier run, but the + iPXE menu boots them. Install linux-image-{{ blancco_kver }} + + linux-modules-{{ blancco_kver }} (offline packages, + scripts/download-packages.sh) or set blancco_kver to an installed + version (ls /lib/modules) AND change the menu to match, then re-run. + when: + - not blancco_kernel_ok + - not (blancco_served.results | map(attribute='stat.exists') | min) + + - name: "Blancco: warn when the Blancco kernel is missing (earlier build kept)" debug: msg: >- WARNING: /boot/vmlinuz-{{ blancco_kver }} or /lib/modules/{{ blancco_kver }} is missing, so the Blancco UEFI - files (vmlinuz-ubuntu-{{ blancco_ksuffix }}, - kexec-initrd-{{ blancco_ksuffix }}.img, - kmod-{{ blancco_kver }}.tar.gz) are NOT built. Install + files are NOT rebuilt; the copies from an earlier run stay in + place (they may not match blancco_kver). Install linux-image-{{ blancco_kver }} + linux-modules-{{ blancco_kver }} - (scripts/download-packages.sh, BLANCCO_KVER) or set blancco_kver - to an installed version (ls /lib/modules), then re-run. + and re-run. when: not blancco_kernel_ok - name: "Build Blancco PXE initramfs (kexec-initrd-{{ blancco_ksuffix }}.img)" @@ -1079,8 +1461,9 @@ creates: "{{ web_root }}/blancco/.kexec-initrd-{{ blancco_ksuffix }}.{{ blancco_kver }}.built" when: blancco_kernel_ok shell: | - set -e + set -eo pipefail WORK=$(mktemp -d) + trap 'rm -rf "$WORK" "$OUT.tmp"' EXIT KVER="{{ blancco_kver }}" OUT="{{ web_root }}/blancco/kexec-initrd-{{ blancco_ksuffix }}.img" mkdir -p "$WORK"/{bin,lib/modules/$KVER/kernel,lib64,sbin,usr/share/udhcpc,etc,run,proc,sys,dev} @@ -1172,11 +1555,9 @@ find . | cpio -o -H newc --quiet | gzip -1 > "$OUT.tmp" mv -f "$OUT.tmp" "$OUT" cd / - rm -rf "$WORK" rm -f "{{ web_root }}"/blancco/.kexec-initrd-{{ blancco_ksuffix }}.*.built touch "{{ web_root }}/blancco/.kexec-initrd-{{ blancco_ksuffix }}.{{ blancco_kver }}.built" echo "Built $(basename "$OUT") for $KVER: $(stat -c %s "$OUT") bytes" - ignore_errors: yes - name: "Copy Ubuntu kernel for Blancco PXE boot (vmlinuz-ubuntu-{{ blancco_ksuffix }})" copy: @@ -1312,15 +1693,25 @@ when: usb_images_dir.stat.exists - name: "Restart and enable services" + # Held while imaging is active (see "Re-run safety"): started + + # enabled only, so a WinPE client mid-copy keeps its SMB session. systemd: name: "{{ item }}" - state: restarted + state: "{{ 'started' if pxe_hold_disruptive | bool else 'restarted' }}" enabled: yes loop: - dnsmasq - apache2 - smbd + - name: "Services NOT restarted (imaging in progress)" + debug: + msg: >- + dnsmasq, apache2 and smbd were not restarted because imaging is + active; config changes from this run apply at their next restart. + Re-run when idle (or -e allow_disruptive=true) to restart them. + when: pxe_hold_disruptive | bool + # Firewall per interface. Imaging NIC: everything imaging needs. GE # corporate NIC: SSH and the webapp only. New rules go in first, then the # old any-interface rules are removed, so SSH is never closed mid-run. @@ -1335,7 +1726,7 @@ loop_control: label: "{{ pxe_nic_name }} {{ item.port }}/{{ item.proto }}" - - name: "UFW: allow SSH and the webapp on the GE corporate NIC" + - name: "UFW: allow SSH and the webapp on the GE corporate NIC (from all of GE)" ufw: rule: allow direction: in @@ -1345,6 +1736,70 @@ loop: "{{ ufw_corp_rules }}" loop_control: label: "{{ corp_nic_name }} {{ item.port }}/{{ item.proto }}" + when: corp_admin_cidrs | length == 0 + + - name: "UFW: WARNING, SSH and the root webapp are open to all of GE" + debug: + msg: + - "!!! WARNING: corp_admin_cidrs is empty. SSH (password auth, shared" + - "!!! 'pxe' password) and the webapp on 9009 (no login, runs as root)" + - "!!! accept connections from ANY host on the GE corporate network via" + - "!!! {{ corp_nic_name }}. Set corp_admin_cidrs in playbook/site-vars.yml" + - "!!! to the admin subnets (docs/SITE-CUSTOMIZATION.md) and re-run." + when: corp_admin_cidrs | length == 0 + + - name: "UFW: allow SSH and the webapp on the GE corporate NIC from corp_admin_cidrs only" + ufw: + rule: allow + direction: in + interface: "{{ corp_nic_name }}" + from_ip: "{{ item.1 }}" + port: "{{ item.0.port }}" + proto: "{{ item.0.proto }}" + loop: "{{ ufw_corp_rules | product(corp_admin_cidrs) | list }}" + loop_control: + label: "{{ corp_nic_name }} from {{ item.1 }} {{ item.0.port }}/{{ item.0.proto }}" + when: corp_admin_cidrs | length > 0 + + - name: "UFW: drop the any-source rules on the GE corporate NIC (restricted now)" + ufw: + rule: allow + direction: in + interface: "{{ corp_nic_name }}" + port: "{{ item.port }}" + proto: "{{ item.proto }}" + delete: yes + loop: "{{ ufw_corp_rules }}" + loop_control: + label: "{{ corp_nic_name }} any {{ item.port }}/{{ item.proto }}" + when: corp_admin_cidrs | length > 0 + + # Only drop the old any-interface rules (allow 22 among them) once both + # named NICs exist on the right MACs and the per-interface SSH rules are + # in UFW. Otherwise those rules would name interfaces that are not + # there and removing the old ones would close SSH. + - name: "UFW: verify both named NICs and their per-interface rules" + args: + executable: /bin/bash + shell: | + set -u + bad=0 + for pair in "{{ pxe_nic_name }}={{ pxe_nic_mac | lower }}" "{{ corp_nic_name }}={{ corp_nic_mac | lower }}"; do + n=${pair%%=*}; m=${pair#*=} + if [ "$(cat "/sys/class/net/$n/address" 2>/dev/null)" != "$m" ]; then + echo "ERROR: interface $n with MAC $m is not present" >&2; bad=1 + fi + done + added=$(ufw show added 2>/dev/null) + for n in "{{ pxe_nic_name }}" "{{ corp_nic_name }}"; do + if ! grep -E "allow in on $n( |$)" <<<"$added" | grep -qE 'port 22 proto tcp'; then + echo "ERROR: no UFW 'allow in on $n ... port 22 proto tcp' rule" >&2; bad=1 + fi + done + [ "$bad" -eq 0 ] || { echo "the old any-interface rules are kept" >&2; exit 1; } + echo "both NICs present, per-interface SSH rules in place" + register: ufw_verify + changed_when: false - name: "UFW: remove the old any-interface rules" ufw: @@ -1377,6 +1832,50 @@ state: enabled policy: deny + # --- SSH on the GE NIC: password logins (corp_ssh_password_auth) --- + # The drop-in sorts before 50-cloud-init.conf (autoinstall allow-pw), and + # sshd keeps the first value it reads. "Match all" ends the block so the + # rest of sshd_config is not scoped to it. + - name: "SSH: password logins only from the imaging LAN (corp_ssh_password_auth: false)" + copy: + dest: /etc/ssh/sshd_config.d/10-pxe-corp-auth.conf + mode: '0644' + content: | + # Managed by pxe_server_setup.yml (corp_ssh_password_auth: false). + # Keys only from the GE corporate side; the shared password still + # works from the imaging LAN and localhost. + PasswordAuthentication no + KbdInteractiveAuthentication no + Match Address {{ pxe_subnet }},127.0.0.0/8,::1 + PasswordAuthentication yes + Match all + register: sshd_dropin_add + when: not (corp_ssh_password_auth | bool) + + - name: "SSH: password logins allowed everywhere (corp_ssh_password_auth: true)" + file: + path: /etc/ssh/sshd_config.d/10-pxe-corp-auth.conf + state: absent + register: sshd_dropin_del + when: corp_ssh_password_auth | bool + + - name: "SSH: check the sshd config, reload it" + when: (sshd_dropin_add is changed) or (sshd_dropin_del is changed) + block: + - name: "SSH: sshd -t" + command: /usr/sbin/sshd -t + changed_when: false + - name: "SSH: reload sshd" + command: systemctl try-reload-or-restart ssh.service + rescue: + - name: "SSH: bad config, remove the drop-in again" + file: + path: /etc/ssh/sshd_config.d/10-pxe-corp-auth.conf + state: absent + - name: "SSH: fail" + fail: + msg: "sshd -t rejected the config with the corp auth drop-in; it was removed and sshd not reloaded." + - name: "Schedule dnsmasq restart 15s after reboot" copy: dest: /etc/cron.d/dnsmasq-restart @@ -1447,19 +1946,23 @@ - name: "Copy webapp from USB" # The whole tree (app, config, services, views, templates, static), every # run, so a re-run with a newer USB updates the app. Was: cp with - # creates: app.py, which never updated an existing install. + # creates: app.py, which never updated an existing install. -i lists + # what changed, so the service restarts only then. shell: | - set -e + set -eo pipefail test -f "{{ usb_root }}/webapp/app.py" - rsync -a --delete --exclude __pycache__ "{{ usb_root }}/webapp/" /opt/pxe-webapp/ + rsync -ai --delete --exclude __pycache__ "{{ usb_root }}/webapp/" /opt/pxe-webapp/ args: executable: /bin/bash + register: webapp_rsync + changed_when: webapp_rsync.stdout | trim | length > 0 - name: "Install webapp Python dependencies (offline wheels)" args: executable: /bin/bash shell: | - # Find the pip-wheels directory on the CIDATA mount + # Find the pip-wheels directory on the CIDATA mount. No online + # fallback: an unpinned pip install from PyPI is never wanted here. export WHEEL_DIR="" for d in "{{ usb_root }}/pip-wheels" "{{ usb_mount }}/pip-wheels"; do if [ -d "$d" ] && compgen -G "$d/*.whl" > /dev/null; then @@ -1467,41 +1970,50 @@ break fi done - if [ -n "$WHEEL_DIR" ]; then - # Install wheels directly using Python zipfile (bypasses pip entirely) - # This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12 - export REQS="{{ usb_root }}/webapp/requirements.txt" - python3 << 'PYEOF' - # Installs only the pinned wheels: the sha256 of every wheel must be in - # webapp/requirements.txt, and each pinned package must have exactly one - # wheel. An older installed version of a package is removed first (by its - # RECORD), so an upgrade does not leave the old files beside the new. + if [ -z "$WHEEL_DIR" ]; then + echo "ERROR: no pip-wheels/*.whl under {{ usb_root }} or {{ usb_mount }}." >&2 + echo "Build them with scripts/download-packages.sh and rebuild the USB." >&2 + exit 1 + fi + # Install wheels directly using Python zipfile (bypasses pip entirely) + # This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12 + export REQS="{{ usb_root }}/webapp/requirements.txt" + python3 << 'PYEOF' + # Installs only the pinned wheels: the sha256 of each wheel must be one of + # the hashes pinned for THAT package in webapp/requirements.txt, and + # each pinned package must have exactly one wheel. A package already + # installed at the pinned version is left alone; an older installed + # version is removed first (by its RECORD), so an upgrade does not + # leave the old files beside the new. Last line CHANGED / UNCHANGED. import zipfile, sysconfig, glob, os, re, sys, hashlib, csv, shutil + norm = lambda n: re.sub(r'[-_.]+', '_', n).lower() site = sysconfig.get_path('platlib') wheel_dir = os.environ['WHEEL_DIR'] reqs = open(os.environ['REQS']).read() - pins, hashes = {}, set() + pins, hashes = {}, {} for m in re.finditer(r'^([A-Za-z0-9_.-]+)==([^\s\\]+)((?:\s*\\?\s*--hash=sha256:[0-9a-f]{64})+)', reqs, re.M): - name = re.sub(r'[-_.]+', '_', m.group(1)).lower() + name = norm(m.group(1)) pins[name] = m.group(2) - hashes.update(re.findall(r'sha256:([0-9a-f]{64})', m.group(3))) + hashes[name] = set(re.findall(r'sha256:([0-9a-f]{64})', m.group(3))) wheels = sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))) seen = {} for whl in wheels: h = hashlib.sha256(open(whl, 'rb').read()).hexdigest() - name = re.sub(r'[-_.]+', '_', os.path.basename(whl).split('-')[0]).lower() - if h not in hashes or name not in pins: - sys.exit(f'REFUSED: {os.path.basename(whl)} is not a pinned wheel (sha256 {h[:16]}...)') + parts = os.path.basename(whl).split('-') + name = norm(parts[0]) + if name not in pins or h not in hashes[name] or parts[1] != pins[name]: + sys.exit(f'REFUSED: {os.path.basename(whl)} is not the pinned wheel of its package (sha256 {h[:16]}...)') seen.setdefault(name, []).append(whl) bad = [n for n in pins if len(seen.get(n, [])) != 1] if bad: sys.exit(f'REFUSED: need exactly one wheel for each pinned package; check: {", ".join(bad)}') + changed = False for name, (whl,) in sorted(seen.items()): - for old in glob.glob(os.path.join(site, '*.dist-info')): - stem = os.path.basename(old)[:-len('.dist-info')] - oname = re.sub(r'[-_.]+', '_', stem.rsplit('-', 1)[0]).lower() - if oname != name: - continue + current = [d for d in glob.glob(os.path.join(site, '*.dist-info')) + if norm(os.path.basename(d)[:-len('.dist-info')].rsplit('-', 1)[0]) == name] + if [os.path.basename(d)[:-len('.dist-info')].rsplit('-', 1)[1] for d in current] == [pins[name]]: + continue + for old in current: rec = os.path.join(old, 'RECORD') if os.path.isfile(rec): for row in csv.reader(open(rec)): @@ -1514,13 +2026,12 @@ print(f' Installing {os.path.basename(whl)}') with zipfile.ZipFile(whl) as z: z.extractall(site) + changed = True print('All pinned wheels installed to ' + site) + print('CHANGED' if changed else 'UNCHANGED') PYEOF - else - # Fallback: try system pip (works if system has internet and compatible pip) - python3 -m pip install --break-system-packages flask lxml 2>/dev/null || - pip3 install --break-system-packages flask lxml - fi + register: webapp_wheels + changed_when: webapp_wheels.stdout_lines | last | default('') == 'CHANGED' - name: "Create systemd service for PXE webapp" copy: @@ -1551,6 +2062,7 @@ [Install] WantedBy=multi-user.target + register: webapp_unit - name: "Enable and start PXE webapp service" systemd: @@ -1558,6 +2070,17 @@ state: started enabled: yes daemon_reload: yes + register: webapp_started + + - name: "Restart PXE webapp (new code, wheels or unit on this run)" + # A re-run used to leave the old code running until the next reboot. + systemd: + name: pxe-webapp + state: restarted + daemon_reload: yes + when: + - webapp_rsync is changed or webapp_wheels is changed or webapp_unit is changed + - webapp_started is not changed # Monthly Blancco CMMC evidence pack (ENT-056/158): on the 1st, save last # month's ZIP to the reports share's evidence/ folder for the tech to copy @@ -1602,9 +2125,8 @@ dest: /etc/apache2/sites-available/pxe-server.conf content: | # 9009 listens on every address: admins use the webapp from the GE - # corporate NIC. TODO: the webapp has NO authentication; anyone on - # the GE network who can reach this box can drive it. Add auth (or - # restrict 9009 on the corp NIC to an admin subnet in UFW). + # corporate NIC. The webapp has NO authentication: restrict 9009 + # (and 22) on the corp NIC with corp_admin_cidrs (UFW). TODO: auth. Listen 9009 DocumentRoot {{ web_root }} @@ -1657,3 +2179,8 @@ handlers: - name: "reload sysctl" command: sysctl --system + + - name: "restart networkd-dispatcher" + systemd: + name: networkd-dispatcher + state: restarted diff --git a/playbook/site-vars.example.yml b/playbook/site-vars.example.yml index 0b1c129..5bfab75 100644 --- a/playbook/site-vars.example.yml +++ b/playbook/site-vars.example.yml @@ -17,6 +17,12 @@ corp_nic_mac: "" # e.g. "20:88:10:af:0d:e5" # pxe_nic_name: enx34c8d6b11010 # corp_nic_name: enp0s31f6 +# Recommended: who may reach SSH and the webapp (9009) on the corp NIC. +# Empty = all of GE (a warning every run). See docs/SITE-CUSTOMIZATION.md. +# corp_admin_cidrs: ["10.134.48.0/24"] +# Keys only from the GE side (needs ~pxe/.ssh/authorized_keys first): +# corp_ssh_password_auth: false + # Optional: imaging LAN addressing (defaults shown). 172.16.9.1 is also # hard-coded in the iPXE menu, startnet.cmd, blancco-preferences.xml and the # unattend files; see docs/SITE-CUSTOMIZATION.md before changing it. diff --git a/playbook/templates/netplan-pxe.yaml.j2 b/playbook/templates/netplan-pxe.yaml.j2 index bd54bb0..1c0ecf3 100644 --- a/playbook/templates/netplan-pxe.yaml.j2 +++ b/playbook/templates/netplan-pxe.yaml.j2 @@ -5,7 +5,11 @@ # # Two NICs, each matched by MAC and given a stable name: # pxe-lan imaging LAN, static, no gateway, no DNS. The ONLY NIC that -# serves DHCP/TFTP/HTTP boot/SMB. +# serves DHCP/TFTP/HTTP boot/SMB. ignore-carrier: the address +# is put on (and kept) without link, so a switch that is off +# or a cable pulled at boot does not leave dnsmasq, Samba and +# Apache without {{ pxe_server_ip }} (networkd +# ConfigureWithoutCarrier= + IgnoreCarrierLoss=). # ge-corp GE corporate network, DHCP client (SSH + webapp 9009, updates). network: version: 2 @@ -17,6 +21,8 @@ network: set-name: {{ pxe_nic_name }} dhcp4: false dhcp6: false + ignore-carrier: true + optional: true addresses: - {{ pxe_server_ip }}/{{ pxe_prefix }} ge-corp: diff --git a/scripts/build-proxmox-iso.sh b/scripts/build-proxmox-iso.sh index c925540..7b0e441 100755 --- a/scripts/build-proxmox-iso.sh +++ b/scripts/build-proxmox-iso.sh @@ -1,360 +1,28 @@ #!/bin/bash # -# build-proxmox-iso.sh — Build a self-contained PXE server installer ISO for Proxmox +# build-proxmox-iso.sh - UNSUPPORTED, kept only to say what to use instead. # -# Repackages the Ubuntu 24.04 Server ISO with: -# - Autoinstall configuration (zero-touch install) -# - All offline .deb packages and Python wheels -# - Ansible playbook, Flask webapp, and boot tools +# This script built a second, hand-maintained copy of the installer +# (its own late-commands and first-boot script). It drifted from +# autoinstall/user-data: it kept the dead-mirror apt block but dropped the +# step that writes the real archive sources back, installed the debs with +# "dpkg -i ... || true" instead of the checked offline-apt-install.sh path, +# had no site vars (so the two-NIC playbook always stopped), and no +# first-boot failure handling. Rather than keep two installers in step, +# there is one: build-usb.sh. Its image mode needs no root and gives a disk +# image a Proxmox VM boots directly: # -# The resulting ISO can be uploaded to Proxmox, attached to a VM, and booted. -# Ubuntu auto-installs, then first-boot configures all PXE services automatically. +# scripts/build-usb.sh --site-vars site-vars.yml --image pxe-usb.img auto \ +# ubuntu-24.04.x-live-server-amd64.iso +# # on the Proxmox host: +# qm importdisk pxe-usb.img +# # attach it as a second disk (or USB), boot order: system disk, then it # -# Usage: -# ./build-proxmox-iso.sh /path/to/ubuntu-24.04-live-server-amd64.iso [output.iso] -# -# Prerequisites (on build workstation): -# sudo apt install xorriso p7zip-full -# -# Before building, run: -# ./download-packages.sh (downloads offline .debs + pip wheels) -# ./prepare-boot-tools.sh ... (extracts Clonezilla, Blancco, Memtest) +# The VM needs two NICs (imaging bridge with no other DHCP server, and the +# corp/GE bridge); put their MACs in site-vars.yml. tests/vm/ does the same +# with libvirt. -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "$0")"/.. && pwd)" -AUTOINSTALL_DIR="$REPO_ROOT/autoinstall" -PLAYBOOK_DIR="$REPO_ROOT/playbook" -OFFLINE_PKG_DIR="$REPO_ROOT/offline-packages" -WEBAPP_DIR="$REPO_ROOT/webapp" -PIP_WHEELS_DIR="$REPO_ROOT/pip-wheels" -BOOT_TOOLS_DIR="$REPO_ROOT/boot-tools" - -# --- Validate arguments --- -if [ $# -lt 1 ]; then - echo "Usage: $0 /path/to/ubuntu-24.04-live-server-amd64.iso [output.iso]" - echo "" - echo " Creates a self-contained ISO for deploying the PXE server in Proxmox." - echo " The ISO auto-installs Ubuntu and configures all PXE services." - echo "" - echo "Prerequisites:" - echo " sudo apt install xorriso p7zip-full" - exit 1 -fi - -UBUNTU_ISO="$(realpath "$1")" -OUTPUT_ISO="${2:-$REPO_ROOT/pxe-server-proxmox.iso}" - -# --- Validate prerequisites --- -echo "============================================" -echo "PXE Server Proxmox ISO Builder" -echo "============================================" -echo "" - -MISSING_CMDS=() -for cmd in xorriso 7z; do - if ! command -v "$cmd" &>/dev/null; then - MISSING_CMDS+=("$cmd") - fi -done - -if [ ${#MISSING_CMDS[@]} -gt 0 ]; then - echo "ERROR: Missing required tools: ${MISSING_CMDS[*]}" - echo "Install with: sudo apt install xorriso p7zip-full" - exit 1 -fi - -if [ ! -f "$UBUNTU_ISO" ]; then - echo "ERROR: ISO not found at $UBUNTU_ISO" - exit 1 -fi - -# Quick sanity check: ensure it looks like an Ubuntu Server ISO -ISO_CONTENTS=$(7z l "$UBUNTU_ISO" 2>&1) || true -if ! echo "$ISO_CONTENTS" | grep -q "casper/vmlinuz"; then - echo "ERROR: Does not appear to be an Ubuntu Server ISO (missing casper/vmlinuz)" - exit 1 -fi - -if [ ! -f "$AUTOINSTALL_DIR/user-data" ]; then - echo "ERROR: user-data not found at $AUTOINSTALL_DIR/user-data" - exit 1 -fi - -if [ ! -f "$PLAYBOOK_DIR/pxe_server_setup.yml" ]; then - echo "ERROR: pxe_server_setup.yml not found at $PLAYBOOK_DIR/" - exit 1 -fi - -echo "Ubuntu ISO : $UBUNTU_ISO" -echo "Output ISO : $OUTPUT_ISO" -echo "Source Dir : $REPO_ROOT" -echo "" - -# --- Setup work directory with cleanup trap --- -WORK_DIR=$(mktemp -d) -cleanup() { rm -rf "$WORK_DIR"; } -trap cleanup EXIT - -EXTRACT_DIR="$WORK_DIR/iso" -BOOT_IMG_DIR="$WORK_DIR/BOOT" - -# --- Step 1: Extract Ubuntu ISO --- -echo "[1/6] Extracting Ubuntu ISO..." -7z x -o"$EXTRACT_DIR" "$UBUNTU_ISO" -y >/dev/null 2>&1 - -# 7z extracts [BOOT] directory containing EFI images needed for rebuild -# Move it out so it doesn't end up in the final ISO filesystem -if [ -d "$EXTRACT_DIR/[BOOT]" ]; then - mv "$EXTRACT_DIR/[BOOT]" "$BOOT_IMG_DIR" - echo " Extracted boot images for BIOS + UEFI" -else - echo "ERROR: [BOOT] directory not found in extracted ISO" - echo " The Ubuntu ISO may be corrupted or an unsupported version." - exit 1 -fi - -# Ensure files are writable (ISO extraction may set read-only) -chmod -R u+w "$EXTRACT_DIR" - -# --- Step 2: Generate autoinstall user-data --- -echo "[2/6] Generating autoinstall configuration..." -mkdir -p "$EXTRACT_DIR/server" -touch "$EXTRACT_DIR/server/meta-data" - -# Reuse the common sections (identity, network, storage, SSH) from existing user-data -# and replace late-commands with ISO-specific versions -sed '/^ late-commands:/,$d' "$AUTOINSTALL_DIR/user-data" > "$EXTRACT_DIR/server/user-data" - -# Append ISO-specific late-commands -cat >> "$EXTRACT_DIR/server/user-data" << 'LATE_COMMANDS' - late-commands: - # Copy project files from ISO (/cdrom/pxe-data/) to the installed system - - mkdir -p /target/opt/pxe-setup - - cp -r /cdrom/pxe-data/packages /target/opt/pxe-setup/ 2>/dev/null || true - - cp -r /cdrom/pxe-data/playbook /target/opt/pxe-setup/ 2>/dev/null || true - - cp -r /cdrom/pxe-data/webapp /target/opt/pxe-setup/ 2>/dev/null || true - - cp -r /cdrom/pxe-data/pip-wheels /target/opt/pxe-setup/ 2>/dev/null || true - - cp -r /cdrom/pxe-data/boot-tools /target/opt/pxe-setup/ 2>/dev/null || true - # Copy boot files (wimboot, boot.wim, BCD, ipxe.efi, etc.) from pxe-data root - - sh -c 'for f in /cdrom/pxe-data/*; do [ -f "$f" ] && cp "$f" /target/opt/pxe-setup/; done' || true - - # Install deb packages in target chroot - - | - curtin in-target --target=/target -- bash -c ' - if compgen -G "/opt/pxe-setup/packages/*.deb" > /dev/null; then - dpkg -i /opt/pxe-setup/packages/*.deb 2>/dev/null || true - dpkg -i /opt/pxe-setup/packages/*.deb 2>/dev/null || true - if command -v nmcli >/dev/null; then - systemctl enable NetworkManager - fi - fi - ' - - # Create first-boot script (reads from local /opt/pxe-setup/) - - | - curtin in-target --target=/target -- bash -c ' - cat <<"EOF" > /opt/first-boot.sh - #!/bin/bash - SRC=/opt/pxe-setup - # Install all offline .deb packages - if compgen -G "$SRC/packages/*.deb" > /dev/null; then - dpkg -i $SRC/packages/*.deb 2>/dev/null || true - dpkg -i $SRC/packages/*.deb 2>/dev/null || true - fi - # Run the Ansible playbook (override USB paths to local source) - if [ -f $SRC/playbook/pxe_server_setup.yml ]; then - cd $SRC/playbook - ansible-playbook -i localhost, -c local pxe_server_setup.yml \ - -e usb_root=$SRC -e usb_mount=$SRC/playbook - fi - # Disable rc.local to prevent rerunning - sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local - lvextend -r -l +100%FREE /dev/mapper/ubuntu--vg-ubuntu--lv || true - # Clean up large setup files to save disk space - rm -rf $SRC/packages $SRC/pip-wheels $SRC/boot-tools - rm -f $SRC/boot.wim $SRC/boot.sdi $SRC/bootx64.efi $SRC/wimboot $SRC/ipxe.efi $SRC/BCD $SRC/boot.stl - EOF - ' - - curtin in-target --target=/target -- chmod +x /opt/first-boot.sh - - # Create rc.local to run first-boot on next startup - - | - curtin in-target --target=/target -- bash -c ' - cat <<"EOF" > /etc/rc.local - #!/bin/bash - /opt/first-boot.sh > /var/log/first-boot.log 2>&1 & - exit 0 - EOF - ' - - curtin in-target --target=/target -- chmod +x /etc/rc.local - - user-data: - disable_root: false - - refresh-installer: - update: no -LATE_COMMANDS - -echo " Generated server/user-data and server/meta-data" - -# --- Step 3: Copy project files to pxe-data/ --- -echo "[3/6] Copying project files to ISO..." -PXE_DATA="$EXTRACT_DIR/pxe-data" -mkdir -p "$PXE_DATA" - -# Offline .deb packages -if [ -d "$OFFLINE_PKG_DIR" ]; then - mkdir -p "$PXE_DATA/packages" - DEB_COUNT=0 - for deb in "$OFFLINE_PKG_DIR"/*.deb; do - if [ -f "$deb" ]; then - cp "$deb" "$PXE_DATA/packages/" - DEB_COUNT=$((DEB_COUNT + 1)) - fi - done - echo " Copied $DEB_COUNT .deb packages" -else - echo " WARNING: No offline-packages/ directory. Run download-packages.sh first." -fi - -# Ansible playbook -mkdir -p "$PXE_DATA/playbook" -cp "$PLAYBOOK_DIR/"* "$PXE_DATA/playbook/" 2>/dev/null || true -echo " Copied playbook/" - -# Flask webapp -if [ -d "$WEBAPP_DIR" ]; then - # Whole tree: app.py imports config, services/ and views/, and the - # Blancco evidence timer runs blancco_evidence_monthly.py. - rm -rf "$PXE_DATA/webapp" - cp -r "$WEBAPP_DIR" "$PXE_DATA/webapp" - find "$PXE_DATA/webapp" -name __pycache__ -type d -prune -exec rm -rf {} + - find "$PXE_DATA/webapp" -name '*.py[co]' -delete - echo " Copied webapp/" -fi - -# Python wheels -if [ ! -d "$PIP_WHEELS_DIR" ]; then - echo " pip-wheels/ not found — downloading now..." - mkdir -p "$PIP_WHEELS_DIR" - if pip3 download -d "$PIP_WHEELS_DIR" flask lxml 2>/dev/null; then - echo " Downloaded pip wheels successfully." - else - echo " WARNING: Failed to download pip wheels (no internet?)" - rmdir "$PIP_WHEELS_DIR" 2>/dev/null || true - fi -fi -if [ -d "$PIP_WHEELS_DIR" ]; then - cp -r "$PIP_WHEELS_DIR" "$PXE_DATA/pip-wheels" - WHEEL_COUNT=$(find "$PIP_WHEELS_DIR" -name '*.whl' | wc -l) - echo " Copied pip-wheels/ ($WHEEL_COUNT wheels)" -fi - -# WinPE boot files (wimboot, boot.wim, BCD, ipxe.efi, etc.) -BOOT_FILES_DIR="$REPO_ROOT/boot-files" -if [ -d "$BOOT_FILES_DIR" ]; then - BOOT_FILE_COUNT=0 - for bf in "$BOOT_FILES_DIR"/*; do - if [ -f "$bf" ]; then - cp "$bf" "$PXE_DATA/" - BOOT_FILE_COUNT=$((BOOT_FILE_COUNT + 1)) - fi - done - BOOT_FILES_SIZE=$(du -sh "$BOOT_FILES_DIR" | cut -f1) - echo " Copied $BOOT_FILE_COUNT boot files ($BOOT_FILES_SIZE) — wimboot, boot.wim, ipxe.efi, etc." -else - echo " WARNING: No boot-files/ found (copy WinPE boot files from Media Creator)" -fi - -# Boot tools (Clonezilla, Blancco, Memtest) -if [ -d "$BOOT_TOOLS_DIR" ]; then - cp -r "$BOOT_TOOLS_DIR" "$PXE_DATA/boot-tools" - TOOLS_SIZE=$(du -sh "$PXE_DATA/boot-tools" | cut -f1) - echo " Copied boot-tools/ ($TOOLS_SIZE)" -else - echo " No boot-tools/ found (run prepare-boot-tools.sh first)" -fi - -# --- Step 4: Modify GRUB for autoinstall --- -echo "[4/6] Configuring autoinstall boot..." -GRUB_CFG="$EXTRACT_DIR/boot/grub/grub.cfg" - -if [ ! -f "$GRUB_CFG" ]; then - echo "ERROR: boot/grub/grub.cfg not found in extracted ISO" - exit 1 -fi - -# Add autoinstall kernel parameter with nocloud datasource pointing to /cdrom/server/ -# The semicolon must be escaped as \; in GRUB (it's a command separator) -# Apply to both regular and HWE kernels -sed -i 's|/casper/vmlinuz\b|/casper/vmlinuz autoinstall ds=nocloud\\;s=/cdrom/server/|g' "$GRUB_CFG" -sed -i 's|/casper/hwe-vmlinuz\b|/casper/hwe-vmlinuz autoinstall ds=nocloud\\;s=/cdrom/server/|g' "$GRUB_CFG" - -# Reduce timeout for automatic boot (1 second instead of default 30) -sed -i 's/set timeout=.*/set timeout=1/' "$GRUB_CFG" - -echo " Modified GRUB: autoinstall enabled, timeout=1s" - -# --- Step 5: Rebuild ISO --- -echo "[5/6] Rebuilding ISO (this may take a few minutes)..." - -# Verify required boot images exist -EFI_IMG="$BOOT_IMG_DIR/2-Boot-NoEmul.img" -if [ ! -f "$EFI_IMG" ]; then - echo "ERROR: EFI boot image not found at $EFI_IMG" - exit 1 -fi - -if [ ! -f "$EXTRACT_DIR/boot/grub/i386-pc/eltorito.img" ]; then - echo "ERROR: BIOS boot image not found at boot/grub/i386-pc/eltorito.img" - exit 1 -fi - -xorriso -as mkisofs -r \ - -V 'PXE-SERVER' \ - -o "$OUTPUT_ISO" \ - --grub2-mbr --interval:local_fs:0s-15s:zero_mbrpt,zero_gpt:"$UBUNTU_ISO" \ - --protective-msdos-label \ - -partition_cyl_align off \ - -partition_offset 16 \ - --mbr-force-bootable \ - -append_partition 2 28732ac11ff8d211ba4b00a0c93ec93b "$EFI_IMG" \ - -appended_part_as_gpt \ - -iso_mbr_part_type a2a0d0ebe5b9334487c068b6b72699c7 \ - -c '/boot.catalog' \ - -b '/boot/grub/i386-pc/eltorito.img' \ - -no-emul-boot -boot-load-size 4 -boot-info-table --grub2-boot-info \ - -eltorito-alt-boot \ - -e '--interval:appended_partition_2:::' \ - -no-emul-boot \ - "$EXTRACT_DIR" - -# --- Step 6: Done --- -echo "[6/6] Cleaning up..." - -ISO_SIZE=$(du -sh "$OUTPUT_ISO" | cut -f1) - -echo "" -echo "============================================" -echo "Proxmox ISO build complete!" -echo "============================================" -echo "" -echo "Output: $OUTPUT_ISO ($ISO_SIZE)" -echo "" -echo "Proxmox deployment:" -echo " 1. Upload ISO to Proxmox storage (Datacenter -> Storage -> ISO Images)" -echo " 2. Create a new VM:" -echo " - BIOS: OVMF (UEFI) — or SeaBIOS (both work)" -echo " - Memory: 4096 MB" -echo " - CPU: 2+ cores" -echo " - Disk: 40+ GB (VirtIO SCSI)" -echo " - Network: Bridge connected to isolated PXE network" -echo " 3. Attach ISO as CD-ROM and start the VM" -echo " 4. Ubuntu auto-installs (~10-15 minutes, zero interaction)" -echo " 5. After reboot, first-boot configures all PXE services" -echo " 6. Access webapp at http://172.16.9.1:9009" -echo "" -echo "NOTE: The VM's network bridge must be connected to your isolated PXE" -echo " network. The server will use static IP 172.16.9.1/24." -echo "" +echo "build-proxmox-iso.sh is no longer supported (it drifted from autoinstall/user-data)." >&2 +echo "Use: scripts/build-usb.sh --site-vars FILE --image pxe-usb.img auto UBUNTU.iso" >&2 +echo "then on Proxmox: qm importdisk pxe-usb.img (see the header of this script)." >&2 +exit 1 diff --git a/scripts/build-usb.sh b/scripts/build-usb.sh index 36e21ca..96fbf30 100755 --- a/scripts/build-usb.sh +++ b/scripts/build-usb.sh @@ -32,9 +32,18 @@ # --skip-boot-tools do not copy boot-tools/ # --site-vars FILE put FILE on CIDATA as playbook/site-vars.yml (the # per-site NIC MACs; see playbook/site-vars.example.yml) +# --no-site-vars build without site vars (the first-boot playbook then +# stops until they are set on the server). One of the +# two is required; playbook/site-vars.yml in the repo +# is never copied implicitly. # --yes do not ask for confirmation (device mode still asks # YES for a non-removable device) # +# Refuses to build when a payload does not match its pin: the iPXE files in +# boot-files/ against the playbook's ipxe_*_sha256, every wheel in +# pip-wheels/ against its own package's hashes in webapp/requirements.txt, +# and the ISO against the volume id offline-packages/ was resolved for. +# # WARNING: device mode ERASES the target USB device. set -euo pipefail @@ -48,8 +57,8 @@ usage() { echo "Usage: sudo $0 [options] /dev/sdX /path/to/ubuntu-24.04.iso [/path/to/winpe-images]" echo " $0 [options] --image out.img SIZE|auto /path/to/ubuntu-24.04.iso [/path/to/winpe-images]" echo "" - echo " Options: --payload-root DIR, --site-vars FILE, --skip-enrollment, --skip-bios," - echo " --skip-drivers, --skip-boot-tools, --yes" + echo " Options: --payload-root DIR, --site-vars FILE | --no-site-vars, --skip-enrollment," + echo " --skip-bios, --skip-drivers, --skip-boot-tools, --yes" echo "" echo " The optional last argument is the path to WinPE deployment content" echo " (e.g., the mounted Media Creator LITE USB). If provided, the images" @@ -70,6 +79,7 @@ SKIP_DRIVERS=0 SKIP_BOOT_TOOLS=0 ASSUME_YES=0 SITE_VARS="" +NO_SITE_VARS=0 POSITIONAL=() while [ $# -gt 0 ]; do case "$1" in @@ -82,6 +92,7 @@ while [ $# -gt 0 ]; do --site-vars) [ $# -ge 2 ] || { usage; exit 1; } SITE_VARS="$2"; shift 2 ;; + --no-site-vars) NO_SITE_VARS=1; shift ;; --skip-enrollment) SKIP_ENROLLMENT=1; shift ;; --skip-bios) SKIP_BIOS=1; shift ;; --skip-drivers) SKIP_DRIVERS=1; shift ;; @@ -181,10 +192,112 @@ for f in "$OFFLINE_PKG_DIR/Packages" "$OFFLINE_PKG_DIR/explicit.list" "$REPO_ROO fi done -if [ -n "$SITE_VARS" ] && [ ! -f "$SITE_VARS" ]; then - echo "ERROR: --site-vars file not found: $SITE_VARS" +if [ -n "$SITE_VARS" ] && [ "$NO_SITE_VARS" -eq 1 ]; then + echo "ERROR: give --site-vars FILE or --no-site-vars, not both" exit 1 fi +if [ -z "$SITE_VARS" ] && [ "$NO_SITE_VARS" -eq 0 ]; then + echo "ERROR: give --site-vars FILE (the server's NIC MACs, see playbook/site-vars.example.yml)" + echo " or --no-site-vars to build a USB whose first boot stops until they are set." + exit 1 +fi +if [ -n "$SITE_VARS" ]; then + if [ ! -f "$SITE_VARS" ]; then + echo "ERROR: --site-vars file not found: $SITE_VARS" + exit 1 + fi + for v in pxe_nic_mac corp_nic_mac; do + if ! grep -qE "^$v:[[:space:]]*[\"']?([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}[\"']?[[:space:]]*(#.*)?$" "$SITE_VARS"; then + echo "ERROR: $SITE_VARS does not set $v to a MAC (aa:bb:cc:dd:ee:ff)" + exit 1 + fi + done +fi + +# --- Payload pins: refuse to build a USB the server would reject --- +# iPXE chain: the same sha256 pins the playbook checks (single source: the +# ipxe_*_sha256 vars in playbook/pxe_server_setup.yml). +pin_of() { + sed -n "s/^[[:space:]]*$1:[[:space:]]*\"\([0-9a-f]\{64\}\)\".*/\1/p" "$PLAYBOOK_DIR/pxe_server_setup.yml" | head -1 +} +sha_ok() { # sha_ok FILE PIN + [ -f "$1" ] && [ "$(sha256sum <"$1" | cut -d' ' -f1)" = "$2" ] +} +for var in ipxe_tarball_sha256 ipxe_shim_sha256 ipxe_snponly_sha256 ipxe_bios_sha256; do + [ -n "$(pin_of "$var")" ] || { echo "ERROR: no $var pin in playbook/pxe_server_setup.yml"; exit 1; } +done +if ! sha_ok "$BOOT_FILES_DIR/ipxeboot.tar.gz" "$(pin_of ipxe_tarball_sha256)"; then + echo "ERROR: $BOOT_FILES_DIR/ipxeboot.tar.gz is missing or does not match ipxe_tarball_sha256" + echo " (scripts/fetch-ipxe.sh fetches the pinned release)" + exit 1 +fi +if ! sha_ok "$BOOT_FILES_DIR/ipxe-shimx64.efi" "$(pin_of ipxe_shim_sha256)"; then + echo "ERROR: $BOOT_FILES_DIR/ipxe-shimx64.efi is missing or does not match ipxe_shim_sha256" + exit 1 +fi +for pair in "ipxeboot/x86_64-sb/snponly.efi ipxe_snponly_sha256" "ipxeboot/x86_64/ipxe.pxe ipxe_bios_sha256"; do + member=${pair% *}; var=${pair#* } + got=$(tar -xzOf "$BOOT_FILES_DIR/ipxeboot.tar.gz" "$member" 2>/dev/null | sha256sum | cut -d' ' -f1) + if [ "$got" != "$(pin_of "$var")" ]; then + echo "ERROR: $member in ipxeboot.tar.gz does not match $var" + exit 1 + fi +done +echo "iPXE chain matches the playbook pins" + +# Wheels: every wheel must be the pinned version of its package with one +# of THAT package's hashes, and each pinned package needs exactly one. No +# download fallback: the playbook refuses anything else. +if ! compgen -G "$PIP_WHEELS_DIR/*.whl" >/dev/null; then + echo "ERROR: no wheels in $PIP_WHEELS_DIR; build them with scripts/download-packages.sh" + exit 1 +fi +python3 - "$PIP_WHEELS_DIR" "$WEBAPP_DIR/requirements.txt" <<'PY' || exit 1 +import glob, hashlib, os, re, sys +wheel_dir, reqs = sys.argv[1], open(sys.argv[2]).read() +norm = lambda n: re.sub(r"[-_.]+", "_", n).lower() +pins, hashes = {}, {} +for m in re.finditer(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)((?:\s*\\?\s*--hash=sha256:[0-9a-f]{64})+)", reqs, re.M): + pins[norm(m.group(1))] = m.group(2) + hashes[norm(m.group(1))] = set(re.findall(r"sha256:([0-9a-f]{64})", m.group(3))) +if not pins: + sys.exit("ERROR: no hashed pins in webapp/requirements.txt") +seen, bad = {}, [] +for whl in sorted(glob.glob(os.path.join(wheel_dir, "*.whl"))): + base = os.path.basename(whl) + name, ver = norm(base.split("-")[0]), base.split("-")[1] + h = hashlib.sha256(open(whl, "rb").read()).hexdigest() + if name not in pins: + bad.append(f"{base}: {name} is not pinned") + elif ver != pins[name]: + bad.append(f"{base}: version {ver}, pinned {pins[name]}") + elif h not in hashes[name]: + bad.append(f"{base}: sha256 {h[:16]}... is not a pinned hash of {name}") + seen.setdefault(name, []).append(base) +bad += [f"{n}: {len(seen.get(n, []))} wheels (need exactly 1)" for n in pins if len(seen.get(n, [])) != 1] +others = [os.path.basename(p) for p in glob.glob(os.path.join(wheel_dir, "*")) if not p.endswith(".whl")] +bad += [f"{o}: not a wheel" for o in others] +if bad: + print("ERROR: pip-wheels/ does not match webapp/requirements.txt:") + print("\n".join(" " + b for b in bad)) + sys.exit(1) +print(f"{len(seen)} wheels match their pinned hashes") +PY + +# ISO: the offline set is resolved against one ISO's base system. +ISO_VOLID_NOW=$(xorriso -indev "$ISO_PATH" -pvd_info 2>/dev/null | sed -n 's/^Volume Id *: *//p' | head -1) +if [ -f "$OFFLINE_PKG_DIR/iso-volid" ]; then + SET_VOLID=$(head -1 "$OFFLINE_PKG_DIR/iso-volid") + if [ "$SET_VOLID" != "$ISO_VOLID_NOW" ]; then + echo "ERROR: offline-packages/ was built for ISO '$SET_VOLID', this ISO is '$ISO_VOLID_NOW'." + echo " Rebuild it: scripts/download-packages.sh --iso $ISO_PATH $OFFLINE_PKG_DIR" + exit 1 + fi + echo "offline-packages/ matches ISO $ISO_VOLID_NOW" +else + echo "WARNING: $OFFLINE_PKG_DIR/iso-volid missing (set built by an older download-packages.sh);" + echo " cannot check it was resolved against $ISO_VOLID_NOW." +fi # Verify required source files exist for f in "$AUTOINSTALL_DIR/user-data" "$AUTOINSTALL_DIR/meta-data" "$PLAYBOOK_DIR/pxe_server_setup.yml"; do @@ -466,7 +579,7 @@ t_mkdir() { t_cp() { local dest="$1"; shift if [ "$IMAGE_MODE" -eq 0 ]; then - cp -r "$@" "$MOUNT_POINT/$dest/" + cp -rL "$@" "$MOUNT_POINT/$dest/" else mcopy -i "$CIDATA_IMG" -s -Q -D o "$@" "::/${dest:+$dest/}" fi @@ -510,19 +623,27 @@ fi t_cp "" "$REPO_ROOT/scripts/offline-apt-install.sh" echo " Copied $DEB_COUNT .deb packages + Packages + explicit.list to packages/" -# Copy playbook directory -t_cp "" "$PLAYBOOK_DIR" +# Copy playbook directory, without a repo-local site-vars.yml: the site +# vars on the USB are exactly --site-vars (or none with --no-site-vars). +mkdir -p "$WORK/stage" +cp -rL "$PLAYBOOK_DIR" "$WORK/stage/playbook" +rm -f "$WORK/stage/playbook/site-vars.yml" +if [ -n "$SITE_VARS" ]; then + cp "$SITE_VARS" "$WORK/stage/playbook/site-vars.yml" +fi +t_cp "" "$WORK/stage/playbook" +rm -rf "$WORK/stage" echo " Copied playbook/" if [ -n "$SITE_VARS" ]; then - mkdir -p "$WORK/sv" - cp "$SITE_VARS" "$WORK/sv/site-vars.yml" - t_cp playbook "$WORK/sv/site-vars.yml" - rm -rf "$WORK/sv" echo " Copied $SITE_VARS as playbook/site-vars.yml" -elif [ ! -f "$PLAYBOOK_DIR/site-vars.yml" ]; then - echo " WARNING: no playbook/site-vars.yml (and no --site-vars): the playbook" - echo " stops until pxe_nic_mac / corp_nic_mac are set on the server." +else + echo " WARNING: --no-site-vars: the first-boot playbook stops until" + echo " pxe_nic_mac / corp_nic_mac are set on the server." fi +# The unattend linter the playbook runs before it replaces the answer files +# ({{ usb_mount }}/../scripts/lint-unattend.py). +t_mkdir scripts +t_cp scripts "$REPO_ROOT/scripts/lint-unattend.py" # Copy the whole webapp tree (app.py imports config, services/, views/; # the Blancco evidence timer runs blancco_evidence_monthly.py), minus caches. @@ -536,24 +657,10 @@ if [ -d "$WEBAPP_DIR" ]; then echo " Copied webapp/" fi -# Copy pip wheels for offline Flask install -if [ ! -d "$PIP_WHEELS_DIR" ]; then - echo " pip-wheels/ not found - downloading now..." - mkdir -p "$PIP_WHEELS_DIR" - if pip3 download -d "$PIP_WHEELS_DIR" flask lxml 2>/dev/null; then - echo " Downloaded pip wheels successfully." - else - echo " WARNING: Failed to download pip wheels (no internet?)" - echo " The PXE server will need internet to install Flask later," - echo " or manually copy wheels to pip-wheels/ and rebuild." - rmdir "$PIP_WHEELS_DIR" 2>/dev/null || true - fi -fi -if [ -d "$PIP_WHEELS_DIR" ]; then - t_cp "" "$PIP_WHEELS_DIR" - WHEEL_COUNT=$(find -L "$PIP_WHEELS_DIR" -name '*.whl' | wc -l) - echo " Copied pip-wheels/ ($WHEEL_COUNT wheels)" -fi +# Copy the pinned pip wheels (checked against requirements.txt above) +t_cp "" "$PIP_WHEELS_DIR" +WHEEL_COUNT=$(find -L "$PIP_WHEELS_DIR" -name '*.whl' | wc -l) +echo " Copied pip-wheels/ ($WHEEL_COUNT wheels)" # Copy WinPE boot files (wimboot, boot.wim, BCD, ipxe.efi, etc.) to the root if [ -d "$BOOT_FILES_DIR" ]; then @@ -643,8 +750,17 @@ fi # Optionally copy WinPE deployment images if [ -n "$WINPE_SOURCE" ] && [ -d "$WINPE_SOURCE" ]; then echo " Copying WinPE deployment content from $WINPE_SOURCE..." + # FAT32 cannot hold a file of 4 GiB or more, and the playbook's import + # (cp -rn) does not reassemble split parts, so such a file stops here. + BIG=$(find -L "$WINPE_SOURCE" -type f -size +4194303k 2>/dev/null) + if [ -n "$BIG" ]; then + echo "ERROR: WinPE content has files of 4 GiB or more, which FAT32 (CIDATA) cannot hold:" + printf '%s\n' "$BIG" | sed 's/^/ /' + echo " Import those images through the webapp (Images > Import) instead." + exit 1 + fi t_mkdir images - t_cp images "$WINPE_SOURCE"/* || true + t_cp images "$WINPE_SOURCE"/* IMG_SIZE=$(du -shL "$WINPE_SOURCE" | cut -f1) echo " Copied WinPE images ($IMG_SIZE)" elif [ -n "$WINPE_SOURCE" ]; then @@ -689,9 +805,12 @@ if [ "$IMAGE_MODE" -eq 1 ]; then echo " or attach it to a VM as a USB disk (see tests/vm/README.md)." else echo "Next steps:" - echo " 1. Insert USB into target machine" - echo " 2. Boot from USB (F12 / boot menu)" - echo " 3. Ubuntu will auto-install and configure the PXE server" - echo " 4. After reboot, move the NIC to the isolated PXE network" + echo " 1. Cable both NICs: the imaging NIC (pxe_nic_mac) to the isolated" + echo " imaging switch, the corp NIC (corp_nic_mac) to the GE network" + echo " 2. Insert USB into the target machine, boot from it (F12 / boot menu)" + echo " 3. Ubuntu auto-installs; on first boot the playbook pins the NICs by" + echo " MAC (imaging = static 172.16.9.1, corp = DHCP) and sets up PXE" + echo " 4. Check /var/log/first-boot.log; a failure shows a banner at login" + echo " and on the console, and first boot retries at the next reboot" fi echo "" diff --git a/scripts/check-updates.py b/scripts/check-updates.py index 29f7bf5..deb0a74 100755 --- a/scripts/check-updates.py +++ b/scripts/check-updates.py @@ -2,18 +2,21 @@ """check-updates.py - is the offline bundle current? Run before building a USB. * offline-packages/*.deb against Ubuntu 24.04 (noble, noble-updates, - noble-security; main + universe): which have a newer version, and which + noble-security; main, restricted, universe, multiverse): which have a + newer version, and which of those are security updates. * webapp/requirements.txt pins against PyPI (newer release?) and osv.dev (known vulnerabilities in the pinned version). Exit 0 = nothing outstanding; 1 = security updates or vulnerable pins found -(the details are printed); 2 = could not check (no network, bad input). +(the details are printed); 2 = could not check (no network, bad or missing +input such as a folder with no .debs, an unreadable index or API answer). Needs internet (Ubuntu archive, pypi.org, api.osv.dev); read-only. Usage: scripts/check-updates.py [--debs DIR] [--reqs FILE] [--all] """ import argparse +import functools import glob import gzip import json @@ -26,7 +29,11 @@ import urllib.request ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ARCHIVE = "http://archive.ubuntu.com/ubuntu/dists" SUITES = ("noble", "noble-updates", "noble-security") -COMPONENTS = ("main", "universe") +COMPONENTS = ("main", "restricted", "universe", "multiverse") + + +class CheckError(Exception): + """Could not check (exit 2).""" def fetch(url, timeout=60): @@ -34,21 +41,39 @@ def fetch(url, timeout=60): return r.read() +try: + import apt_pkg # python3-apt: in-process comparison, no fork per pair + apt_pkg.init_system() +except Exception: # not installed: fall back to dpkg, cached + apt_pkg = None + + +@functools.lru_cache(maxsize=None) def newer(a, b): """True if Debian version a > b.""" + if a == b: + return False + if apt_pkg is not None: + return apt_pkg.version_compare(a, b) > 0 return subprocess.run(["dpkg", "--compare-versions", a, "gt", b]).returncode == 0 -def ubuntu_index(): - """({package: newest version}, {package: newest security version}).""" +def ubuntu_index(wanted): + """({package: newest version}, {package: newest security version}), + for the package names in wanted only.""" best, sec = {}, {} for suite in SUITES: for comp in COMPONENTS: - text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace") + try: + text = gzip.decompress(fetch(f"{ARCHIVE}/{suite}/{comp}/binary-amd64/Packages.gz")).decode("utf-8", "replace") + except (OSError, EOFError) as ex: + raise CheckError(f"{suite}/{comp} index: {ex}") pkg = None for line in text.splitlines(): if line.startswith("Package: "): pkg = line[9:].strip() + if pkg not in wanted: + pkg = None elif line.startswith("Version: ") and pkg: v = line[9:].strip() if pkg not in best or newer(v, best[pkg]): @@ -61,9 +86,9 @@ def ubuntu_index(): def check_debs(deb_dir, show_all): debs = glob.glob(os.path.join(deb_dir, "*.deb")) if not debs: - print(f"no .deb files in {deb_dir}: nothing to check (a failed download?)") - return 1 - best, sec = ubuntu_index() + raise CheckError(f"no .deb files in {deb_dir}: nothing to check (a failed download?)") + names = {os.path.basename(d).split("_")[0] for d in debs} + best, sec = ubuntu_index(names) stale, security = [], [] for d in debs: parts = os.path.basename(d)[:-4].split("_") @@ -82,14 +107,23 @@ def check_debs(deb_dir, show_all): def check_pins(reqs): - pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", open(reqs).read(), re.M) + try: + text = open(reqs).read() + except OSError as ex: + raise CheckError(f"{reqs}: {ex}") + pins = re.findall(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", text, re.M) if not pins: print(f"no pinned packages in {reqs}") return 0 q = {"queries": [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in pins]} req = urllib.request.Request("https://api.osv.dev/v1/querybatch", data=json.dumps(q).encode(), headers={"Content-Type": "application/json"}) - results = json.load(urllib.request.urlopen(req, timeout=60))["results"] + try: + results = json.load(urllib.request.urlopen(req, timeout=60))["results"] + if not isinstance(results, list) or len(results) != len(pins): + raise ValueError("osv.dev returned %r results for %d pins" % (type(results).__name__, len(pins))) + except (ValueError, KeyError, TypeError) as ex: + raise CheckError(f"osv.dev answer: {ex}") bad = 0 print(f"Python pins in {os.path.relpath(reqs, ROOT)}:") for (n, v), r in zip(pins, results): @@ -97,7 +131,7 @@ def check_pins(reqs): latest = json.loads(fetch(f"https://pypi.org/pypi/{n}/json", 30))["info"]["version"] except Exception: # report, keep going latest = "?" - vulns = [x["id"] for x in r.get("vulns", [])] + vulns = [x.get("id", "?") for x in (r or {}).get("vulns", [])] bad += bool(vulns) flag = "VULNERABLE" if vulns else ("newer " if latest not in (v, "?") else "ok ") print(f" {flag} {n:14} {v:10} latest {latest:10} {', '.join(vulns)}") @@ -112,7 +146,7 @@ def main(): a = ap.parse_args() try: problems = check_debs(a.debs, a.all) + check_pins(a.reqs) - except OSError as ex: + except (OSError, CheckError) as ex: print(f"could not check: {ex}", file=sys.stderr) return 2 print("PASSED: nothing outstanding" if not problems else f"ATTENTION: {problems} item(s) need updating") diff --git a/scripts/deploy-unattend.sh b/scripts/deploy-unattend.sh index add7b4c..c2eff73 100755 --- a/scripts/deploy-unattend.sh +++ b/scripts/deploy-unattend.sh @@ -4,14 +4,17 @@ # server. A bad answer file does not fail at copy time: every bay imaged # with it stops at "Windows could not parse or process unattend answer # file" and has to be re-imaged. So this refuses to copy unless the -# linter passes, backs the live file up, swaps it in atomically, and lints -# the copy on the server afterwards. +# linter passes, backs the live file up, swaps it in atomically, checks +# the live copy is byte-identical (sha256) to the linted one, and lints the +# copy on the server afterwards. Any check failing restores the backup. # # Usage: scripts/deploy-unattend.sh # e.g. scripts/deploy-unattend.sh gea-engineer playbook/FlatUnattendW10-engineer.xml # # Env: PXE_SERVER (default 172.16.9.1), PXE_USER (default pxe); the -# remote password is read from PXE_PASS or prompted for. +# remote password is read from PXE_PASS or prompted for. It never appears +# on a command line (local or remote): sshpass gets it from SSHPASS, sudo +# on the server from stdin. set -euo pipefail @@ -27,34 +30,75 @@ STAMP="$(date +%Y%m%d-%H%M%S)" case "$IMAGE" in *[!A-Za-z0-9._-]*|"") echo "bad image type: $IMAGE" >&2; exit 2 ;; esac +case "$SERVER$USER_" in + *[!A-Za-z0-9._@:-]*) echo "bad PXE_SERVER / PXE_USER" >&2; exit 2 ;; +esac [ -f "$SRC" ] || { echo "no such file: $SRC" >&2; exit 2; } -echo "== 1/3 lint $SRC" +echo "== 1/4 lint $SRC" if ! python3 "$LINT" --local "$SRC"; then echo "REFUSED: the linter failed; nothing was copied." >&2 exit 1 fi +LOCAL_SHA=$(sha256sum <"$SRC" | cut -d' ' -f1) if [ -z "${PXE_PASS:-}" ]; then read -rs -p "password for $USER_@$SERVER: " PXE_PASS; echo fi -export SSHPASS="$PXE_PASS" +export SSHPASS="$PXE_PASS" PXE_PASS PXE_USER="$USER_" PXE_SERVER="$SERVER" SSH=(sshpass -e ssh -o StrictHostKeyChecking=no -o ConnectTimeout=8 "$USER_@$SERVER") -echo "== 2/3 back up and swap $DEST on $SERVER" -sshpass -e scp -q -o StrictHostKeyChecking=no "$SRC" "$USER_@$SERVER:/tmp/unattend-$STAMP.xml" -"${SSH[@]}" "set -e; P() { echo \"\$SSHPASS_REMOTE\" | sudo -S -p '' \"\$@\"; }; SSHPASS_REMOTE='$PXE_PASS' - [ -f '$DEST' ] || { echo 'no live file at $DEST' >&2; exit 1; } - P cp -a '$DEST' '$DEST.bak-$STAMP' - P install -m 644 -o root -g root /tmp/unattend-$STAMP.xml '$DEST.new' - P mv -f '$DEST.new' '$DEST' - rm -f /tmp/unattend-$STAMP.xml - echo \"backup: $DEST.bak-$STAMP\"; sha256sum '$DEST'" +# remote SCRIPT ARGS...: run SCRIPT under bash on the server with ARGS. The +# first stdin line is the password; the script reads it into $P and hands +# it to "sudo -S" through a pipe (printf is a builtin: not in ps). Every +# word is %q-quoted for the remote shell. +remote() { + local script=$1; shift + local cmd + cmd="bash -c $(printf '%q' "$script") _$(printf ' %q' "$@")" + printf '%s\n' "$PXE_PASS" | "${SSH[@]}" "$cmd" +} +# Prologue every remote script starts with. +PRE='set -euo pipefail; IFS= read -r P; S() { printf "%s\n" "$P" | sudo -k -S -p "" "$@"; };' -echo "== 3/3 lint the live copy" -if ! python3 "$LINT" --server "$SERVER" --image "$IMAGE"; then - echo "LIVE FILE FAILS LINT: restoring the backup." >&2 - "${SSH[@]}" "echo '$PXE_PASS' | sudo -S -p '' cp -a '$DEST.bak-$STAMP' '$DEST'" +echo "== 2/4 upload to $SERVER" +TMP=$(remote "$PRE"' mktemp /tmp/unattend.XXXXXX') +case "$TMP" in /tmp/unattend.*) ;; *) echo "remote mktemp failed: $TMP" >&2; exit 1 ;; esac +sshpass -e scp -q -o StrictHostKeyChecking=no "$SRC" "$USER_@$SERVER:$TMP" + +echo "== 3/4 back up and swap $DEST" +OUT=$(remote "$PRE"' + DEST=$1 TMP=$2 STAMP=$3 WANT=$4 + [ -f "$DEST" ] || { echo "no live file at $DEST" >&2; rm -f "$TMP"; exit 1; } + GOT=$(sha256sum <"$TMP" | cut -d" " -f1) + [ "$GOT" = "$WANT" ] || { echo "upload corrupted: $GOT != $WANT" >&2; rm -f "$TMP"; exit 1; } + S cp -a "$DEST" "$DEST.bak-$STAMP" + S install -m 644 -o root -g root "$TMP" "$DEST.new" + S mv -f "$DEST.new" "$DEST" + rm -f "$TMP" + echo "backup: $DEST.bak-$STAMP" + echo "live-sha256: $(sha256sum <"$DEST" | cut -d" " -f1)"' "$DEST" "$TMP" "$STAMP" "$LOCAL_SHA") +echo "$OUT" + +rollback() { + echo "$1: restoring the backup." >&2 + remote "$PRE"' S cp -a "$1.bak-$2" "$1"' "$DEST" "$STAMP" exit 1 +} + +LIVE_SHA=$(sed -n 's/^live-sha256: //p' <<<"$OUT") +if [ "$LIVE_SHA" != "$LOCAL_SHA" ]; then + rollback "LIVE FILE sha256 $LIVE_SHA != linted copy $LOCAL_SHA" fi -echo "Deployed $SRC to $IMAGE." + +echo "== 4/4 lint the live copy" +if ! python3 "$LINT" --server "$SERVER" --image "$IMAGE" --sha256 >"${TMPDIR:-/tmp}/lint-live-$STAMP.log" 2>&1; then + cat "${TMPDIR:-/tmp}/lint-live-$STAMP.log" >&2 + rollback "LIVE FILE FAILS LINT" +fi +cat "${TMPDIR:-/tmp}/lint-live-$STAMP.log" +if ! grep -q "^sha256 $LOCAL_SHA $IMAGE\$" "${TMPDIR:-/tmp}/lint-live-$STAMP.log"; then + rollback "the linter read back a different file than the one deployed" +fi +rm -f "${TMPDIR:-/tmp}/lint-live-$STAMP.log" +echo "Deployed $SRC to $IMAGE (sha256 $LOCAL_SHA)." diff --git a/scripts/download-packages.sh b/scripts/download-packages.sh index c8d6a00..14c7bc4 100755 --- a/scripts/download-packages.sh +++ b/scripts/download-packages.sh @@ -44,13 +44,19 @@ set -euo pipefail SCRIPT_PATH="$(readlink -f "$0")" REPO_DIR="$(cd "$(dirname "$SCRIPT_PATH")"/.. && pwd)" -# Kernel the Blancco UEFI path boots (playbook var blancco_kver; keep the two -# in step). The playbook builds vmlinuz-ubuntu-70, kexec-initrd-70.img and +# Kernel the Blancco UEFI path boots. The default is read from the playbook +# var blancco_kver (the one source; see its comment for the other places). +# The playbook builds vmlinuz-ubuntu-70, kexec-initrd-70.img and # kmod-.tar.gz from this kernel's /boot and /lib/modules, and the # Canonical-signed image is what Ubuntu's shim starts with Secure Boot on. # The versioned packages pin it; the HWE metapackage keeps the box on the # HWE line. Override: BLANCCO_KVER=7.0.0-40-generic ./download-packages.sh -BLANCCO_KVER="${BLANCCO_KVER:-7.0.0-34-generic}" +# (then set blancco_kver to match, or the playbook will not find it). +if [ -z "${BLANCCO_KVER:-}" ]; then + BLANCCO_KVER=$(sed -n 's/^[[:space:]]*blancco_kver:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$REPO_DIR/playbook/pxe_server_setup.yml" | head -1) +fi +[ -n "$BLANCCO_KVER" ] || { echo "ERROR: no blancco_kver in playbook/pxe_server_setup.yml (set BLANCCO_KVER)" >&2; exit 1; } # Packages installed by the Ansible playbook (pxe_server_setup.yml) PLAYBOOK_PACKAGES=( @@ -66,6 +72,7 @@ PLAYBOOK_PACKAGES=( grub-efi-amd64-bin grub-common conntrack + networkd-dispatcher # carrier hook (pxe-nic-carrier); in the base, pinned here anyway busybox-static zstd cpio @@ -300,6 +307,9 @@ docker run --rm \ -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ "$TAG" bash /repo/scripts/download-packages.sh +# The ISO this set was resolved against; build-usb.sh refuses another one. +printf '%s\n' "$VOLID" >"$OUT_DIR_ABS/iso-volid" + # --- 3. offline check: resolve exactly as the server will, no network --- echo "[3/4] Offline check on a fresh base container (no network)..." docker run --rm --network none \ @@ -310,8 +320,15 @@ docker run --rm --network none \ echo "[4/4] Downloading Python wheels for the webapp..." # Exactly the pinned, hashed set in webapp/requirements.txt, for the server's # Python 3.12 on x86_64. The folder is emptied first: the playbook installs -# every wheel in it and refuses any wheel that is not pinned. -find "$PIP_DIR_HOST" -mindepth 1 -delete +# every wheel in it and refuses any wheel that is not pinned. Only a folder +# of wheels is emptied: a PIP_DIR pointing anywhere else is refused. +NOT_WHEELS=$(find "$PIP_DIR_HOST" -mindepth 1 \( ! -name '*.whl' -o ! -type f \) -print -quit) +if [ -n "$NOT_WHEELS" ]; then + echo "ERROR: $PIP_DIR_HOST holds more than *.whl files (e.g. $NOT_WHEELS); refusing to empty it." >&2 + echo " Point PIP_DIR at a wheels-only folder." >&2 + exit 1 +fi +find "$PIP_DIR_HOST" -mindepth 1 -maxdepth 1 -name '*.whl' -type f -delete docker run --rm -v "$REPO_DIR:/repo:ro" -v "$PIP_DIR_HOST:/pipout" \ -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" ubuntu:24.04 bash -euc ' apt-get update -qq >/dev/null diff --git a/scripts/lint-unattend.py b/scripts/lint-unattend.py index 1f378b4..1a70ab1 100755 --- a/scripts/lint-unattend.py +++ b/scripts/lint-unattend.py @@ -38,18 +38,24 @@ Usage: Exits non-zero when any ERROR is found, so it can gate a deploy. +Remote mode reads PXE_SERVER (default 172.16.9.1), PXE_USER (default pxe) +and PXE_PASS (default pxe) from the environment. The password reaches +sshpass through its environment (sshpass -e), never on a command line. + Requires: sshpass (remote mode only) """ import argparse +import os import re +import shlex import subprocess import sys import xml.dom.minidom -PXE_HOST = "172.16.9.1" -PXE_USER = "pxe" -PXE_PASS = "pxe" +PXE_HOST = os.environ.get("PXE_SERVER") or "172.16.9.1" +PXE_USER = os.environ.get("PXE_USER") or "pxe" +PXE_PASS = os.environ.get("PXE_PASS") or "pxe" IMAGE_BASE = "/srv/samba/winpeapps" UNATTEND_REL = "Deploy/FlatUnattendW10.xml" @@ -76,15 +82,16 @@ ERROR, WARN = "ERROR", "WARN" def ssh_cmd(host, cmd): + env = dict(os.environ, SSHPASS=PXE_PASS) return subprocess.run( - ["sshpass", "-p", PXE_PASS, "ssh", "-o", "StrictHostKeyChecking=no", + ["sshpass", "-e", "ssh", "-o", "StrictHostKeyChecking=no", "-o", "LogLevel=ERROR", f"{PXE_USER}@{host}", cmd], - capture_output=True, text=True) + capture_output=True, text=True, env=env) def read_remote_bytes(host, path): """Base64 in transit so a BOM or CRLF survives the hop intact.""" - r = ssh_cmd(host, "base64 -w0 '%s' 2>/dev/null" % path) + r = ssh_cmd(host, "base64 -w0 %s 2>/dev/null" % shlex.quote(path)) if r.returncode != 0 or not r.stdout.strip(): return None import base64 @@ -165,9 +172,13 @@ def main(): parser = argparse.ArgumentParser(description="Validate unattend answer files.") parser.add_argument("--server", default=PXE_HOST, help=f"PXE server (default: {PXE_HOST})") parser.add_argument("--image", help="single image type instead of all") + parser.add_argument("--sha256", action="store_true", + help="also print the sha256 of each file linted (deploy-unattend.sh compares it)") parser.add_argument("--local", nargs="+", metavar="FILE", help="lint local files instead") parser.add_argument("--quiet", action="store_true", help="suppress WARN lines") args = parser.parse_args() + if args.image and not re.fullmatch(r"[A-Za-z0-9._-]+", args.image): + sys.exit("ERROR: bad image type %r" % args.image) findings = [] targets = [] @@ -180,7 +191,7 @@ def main(): images = [args.image] else: r = ssh_cmd(args.server, - "find '%s' -maxdepth 1 -mindepth 1 -type d -printf '%%f\\n'" % IMAGE_BASE) + "find %s -maxdepth 1 -mindepth 1 -type d -printf '%%f\\n'" % shlex.quote(IMAGE_BASE)) if r.returncode != 0: sys.exit("ERROR: cannot list %s: %s" % (IMAGE_BASE, r.stderr.strip())) images = sorted(n for n in r.stdout.split() if not n.startswith("_")) @@ -190,6 +201,9 @@ def main(): for name, data in targets: check(name, data, findings) + if args.sha256 and data is not None: + import hashlib + print("sha256 %s %s" % (hashlib.sha256(data).hexdigest(), name)) print("Linted %d unattend file(s)\n" % len(targets)) shown = [f for f in findings if not (args.quiet and f[0] == WARN)]