Build/playbook re-run safety: corp access, netplan rollback, first-boot retry

H1  corp_admin_cidrs (UFW 22/9009 on the corp NIC from these only; refuses
    when it would lock out the current SSH client) and corp_ssh_password_auth
    (sshd drop-in, passwords only from the imaging subnet). Defaults keep the
    current behaviour and print a loud warning.
H2  pxe-netplan-apply.sh: pre-flight refusal (pxe-MAC NIC holds the default
    route, a DHCP lease or the route to an SSH client), run from the USB
    before anything changes; post-apply checks (names, imaging IPv4, corp
    IPv4 back within 30 s, SSH route) and a 120 s systemd-run auto-revert.
H3  restore() puts the previous NIC names back (two-phase rename via
    temporary names, so swaps work) before generate + apply.
H4  a failed rename is fatal (restore + exit 1); the play drops the old
    any-interface UFW rules only after verifying both NICs and their
    per-interface SSH rules.
H5  first-boot checks the playbook exit code, retries at next boot (max 5),
    writes /var/lib/pxe-firstboot.failed and /etc/issue + /etc/motd banners.
M1  build-usb.sh verifies iPXE files against the playbook pins, every wheel
    against its own package's hashes, drops the unpinned pip fallback,
    requires --site-vars or --no-site-vars.
M2  webapp restarts when the code, wheels or unit changed.
M3  ignore-carrier on the imaging NIC; ip_nonlocal_bind kept for the
    apache/networkd boot race.
M4  menu suffix must match blancco_kver; missing kernel with no earlier
    build is fatal; a live menu with the staged Blancco test entry is not
    overwritten.
M5  build-proxmox-iso.sh is unsupported (hard exit); use build-usb --image.
M6  deploy-unattend.sh: password via SSHPASS env and sudo stdin only,
    %q-quoted remote args, sha256 check of the live file; lint-unattend.py
    reads PXE_SERVER/PXE_USER/PXE_PASS and uses sshpass -e.
M7  the playbook lints the unattend files before the force: yes copies
    (lint-unattend.py ships on CIDATA as scripts/); pre-commit ACMR.
M8  Blancco initramfs build: no ignore_errors, pipefail, trap cleanup.
Also: Send-ImagingStage.ps1 deploy, carrier hook install tasks
(pxe-nic-carrier, networkd-dispatcher), L1 imaging-active hold
(allow_disruptive) and no iPXE downgrade below an Updates-page install,
L2 iso-volid, L3 kver pin read from the playbook, L4 cp -rL, L5 refuse
>= 4 GiB WinPE files, L6 PIP_DIR wipe guard, L7 per-package wheel hashes,
no online fallback, L8 check-updates.py (apt_pkg/cached compare, all
components, exit 2 on no debs / bad answers).

autoinstall/user-data

@@ -89,11 +89,38 @@ autoinstall:
     # Step 2: Disable networkd-wait-online (hangs on air-gapped network)
     - ln -sf /dev/null /target/etc/systemd/system/systemd-networkd-wait-online.service
 
-    # Step 3: Create first-boot.sh
+    # Step 3: Create first-boot.sh. rc.local runs it on every boot until the
+    # playbook succeeds (at most MAX attempts). A failure leaves
+    # /var/lib/pxe-firstboot.failed and a banner on the console (/etc/issue)
+    # and at SSH login (/etc/motd) pointing at /var/log/first-boot.log.
     - |
       curtin in-target --target=/target -- bash -c '
       cat <<"EOF" > /opt/first-boot.sh
       #!/bin/bash
+      MAX=5
+      STATE=/var/lib/pxe-firstboot
+      FAILED=/var/lib/pxe-firstboot.failed
+      mkdir -p "$STATE"
+      N=$(( $(cat "$STATE/attempts" 2>/dev/null || echo 0) + 1 ))
+      echo "$N" > "$STATE/attempts"
+      echo "=== first-boot attempt $N of $MAX, $(date -Is) ==="
+      banner_off() {
+        for f in /etc/issue /etc/motd; do
+          [ -f "$f" ] && sed -i "/^### PXE FIRST-BOOT BEGIN/,/^### PXE FIRST-BOOT END/d" "$f"
+        done
+      }
+      banner_on() {
+        banner_off
+        for f in /etc/issue /etc/motd; do
+          {
+            echo "### PXE FIRST-BOOT BEGIN"
+            echo "!!! PXE server setup (first-boot playbook) FAILED: $1"
+            echo "!!! Log: /var/log/first-boot.log   Marker: $FAILED"
+            echo "!!! $2"
+            echo "### PXE FIRST-BOOT END"
+          } >> "$f"
+        done
+      }
       # Expand root LV to full disk BEFORE playbook (playbook copies ~130GB of drivers+PPKGs)
       ROOT_DEV=$(findmnt -n -o SOURCE /)
       ROOT_DISK=$(lsblk -n -o PKNAME "$(readlink -f "$ROOT_DEV")" | tail -1)
@@ -104,28 +131,52 @@ autoinstall:
         pvresize "$PV_PART" 2>&1 || true
       fi
       lvextend -r -l +100%FREE /dev/mapper/ubuntu--vg-ubuntu--lv 2>&1 || true
+      RC=1
+      WHY="CIDATA (the installer USB) not found"
       CIDATA_DEV=$(blkid -L CIDATA)
       if [ -n "$CIDATA_DEV" ]; then
         mkdir -p /mnt/usb
-        mount "$CIDATA_DEV" /mnt/usb
-        if [ -f /mnt/usb/playbook/pxe_server_setup.yml ]; then
-          cd /mnt/usb/playbook
-          ansible-playbook -i localhost, -c local pxe_server_setup.yml
-          cd /
+        if mount "$CIDATA_DEV" /mnt/usb; then
+          if [ -f /mnt/usb/playbook/pxe_server_setup.yml ]; then
+            cd /mnt/usb/playbook
+            ansible-playbook -i localhost, -c local pxe_server_setup.yml
+            RC=$?
+            WHY="ansible-playbook exit code $RC"
+            cd /
+          else
+            WHY="no playbook/pxe_server_setup.yml on CIDATA"
+          fi
+          umount /mnt/usb
+        else
+          WHY="could not mount CIDATA ($CIDATA_DEV)"
         fi
-        umount /mnt/usb
       fi
-      sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local
+      if [ "$RC" -eq 0 ]; then
+        echo "=== first-boot: playbook succeeded on attempt $N ==="
+        rm -f "$FAILED"
+        banner_off
+        sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local
+        exit 0
+      fi
+      echo "=== first-boot: FAILED ($WHY), attempt $N of $MAX ==="
+      printf "attempt=%s of %s\nwhen=%s\nreason=%s\nlog=/var/log/first-boot.log\n" "$N" "$MAX" "$(date -Is)" "$WHY" > "$FAILED"
+      if [ "$N" -ge "$MAX" ]; then
+        sed -i "s|^/opt/first-boot.sh.*|# &|" /etc/rc.local
+        banner_on "$WHY (attempt $N of $MAX)" "Gave up retrying. Fix it, then run: sudo /opt/first-boot.sh after rm $STATE/attempts"
+      else
+        banner_on "$WHY (attempt $N of $MAX)" "It runs again at the next boot (fix the cause first, e.g. playbook/site-vars.yml on the USB)."
+      fi
+      exit "$RC"
       EOF
       chmod +x /opt/first-boot.sh
       '
 
-    # Step 4: Create rc.local
+    # Step 4: Create rc.local (the log is appended to, one block per attempt)
     - |
       curtin in-target --target=/target -- bash -c '
       cat <<"EOF" > /etc/rc.local
       #!/bin/bash
-      /opt/first-boot.sh > /var/log/first-boot.log 2>&1 &
+      /opt/first-boot.sh >> /var/log/first-boot.log 2>&1 &
       exit 0
       EOF
       chmod +x /etc/rc.local