Fix air-gapped deployment: pip wheel install, UFW ports, installer crash

- Fix pip/distutils incompatibility: install Python wheels directly via
  zipfile extraction instead of broken pip3 from Ubuntu 22.04 .debs
  (pip3 crashes on Python 3.12 with ModuleNotFoundError: distutils)
- Fix UFW port types: quote loop items so string comparison works
  correctly, giving ports 67/69 UDP rules instead of TCP
- Fix autoinstall crash: set refresh-installer to no (can't reach
  internet on air-gapped network, was crashing subiquity)
- Remove python3-pip and python3-venv from download-packages.sh
  (no longer needed with direct wheel extraction)
- Add ignore_errors to WinPE/iPXE copy tasks (files only present
  on real USB media, not test VM)
- Use system python3 instead of venv for webapp service

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

autoinstall/user-data

@@ -9,9 +9,7 @@ autoinstall:
     variant: ""
   timezone: America/New_York
 
-  # Network configuration
+  # Network: static IP for isolated PXE LAN (no internet/DHCP needed)
-  # Uses a broad match so any wired NIC gets the static PXE address.
-  # No WiFi needed — all packages are on the CIDATA partition.
   network:
     version: 2
     ethernets:
@@ -108,4 +106,4 @@ autoinstall:
     disable_root: false
 
   refresh-installer:
-    update: yes
+    update: no

download-packages.sh

@@ -26,8 +26,6 @@ PLAYBOOK_PACKAGES=(
     ufw
     cron
     wimtools
-    python3-pip
-    python3-venv
     p7zip-full
 )
 

playbook/pxe_server_setup.yml

@@ -272,7 +272,7 @@
         - "{{ image_types }}"
         - "{{ deploy_subdirs }}"
 
-    - name: "Copy WinPE & boot files from USB"
+    - name: "Copy WinPE & boot files from USB (skipped if not present)"
       copy:
         src: "{{ usb_mount }}/{{ item.src }}"
         dest: "{{ web_root }}/win11/{{ item.dest }}"
@@ -284,14 +284,16 @@
         - { src: "bootx64.efi", dest: "EFI/Boot/bootx64.efi" }
         - { src: "boot.sdi",    dest: "Boot/boot.sdi" }
         - { src: "boot.wim",    dest: "sources/boot.wim" }
+      ignore_errors: yes
 
-    - name: "Copy iPXE binaries from USB"
+    - name: "Copy iPXE binaries from USB (skipped if not present)"
       copy:
         src: "{{ usb_mount }}/{{ item }}"
         dest: "{{ tftp_dir }}/{{ item }}"
         mode: '0755'
       loop:
         - ipxe.efi
+      ignore_errors: yes
 
     - name: "Copy boot tool files from USB (Clonezilla, Blancco, Memtest)"
       shell: >
@@ -329,12 +331,12 @@
         port: "{{ item }}"
         proto: "{{ 'udp' if item in ['67','69'] else 'tcp' }}"
       loop:
-        - 67
+        - "67"
-        - 69
+        - "69"
-        - 80
+        - "80"
-        - 4433
+        - "4433"
-        - 445
+        - "445"
-        - 9009
+        - "9009"
 
     - name: "Enable UFW firewall"
       ufw:
@@ -362,18 +364,37 @@
       args:
         creates: /opt/pxe-webapp/app.py
 
-    - name: "Create Python virtual environment for webapp"
-      command: python3 -m venv /opt/pxe-webapp/venv
-      args:
-        creates: /opt/pxe-webapp/venv/bin/python
-
     - name: "Install webapp Python dependencies (offline wheels)"
-      shell: >
+      args:
-        /opt/pxe-webapp/venv/bin/pip install --no-index
+        executable: /bin/bash
-        --find-links="{{ usb_mount }}/../pip-wheels/"
+      shell: |
-        --find-links="{{ usb_mount }}/pip-wheels/"
+        # Find the pip-wheels directory on the CIDATA mount
-        -r /opt/pxe-webapp/requirements.txt 2>/dev/null ||
+        export WHEEL_DIR=""
-        /opt/pxe-webapp/venv/bin/pip install -r /opt/pxe-webapp/requirements.txt
+        for d in "{{ usb_mount }}/../pip-wheels" "{{ usb_mount }}/pip-wheels"; do
+          if [ -d "$d" ] && compgen -G "$d/*.whl" > /dev/null; then
+            export WHEEL_DIR="$(cd "$d" && pwd)"
+            break
+          fi
+        done
+        if [ -n "$WHEEL_DIR" ]; then
+          # Install wheels directly using Python zipfile (bypasses pip entirely)
+          # This avoids the pip3/distutils incompatibility between Ubuntu 22.04 debs and Python 3.12
+          python3 << 'PYEOF'
+        import zipfile, sysconfig, glob, os
+        site = sysconfig.get_path('platlib')
+        wheel_dir = os.environ['WHEEL_DIR']
+        for whl in sorted(glob.glob(os.path.join(wheel_dir, '*.whl'))):
+            name = os.path.basename(whl)
+            print(f'  Installing {name}')
+            with zipfile.ZipFile(whl) as z:
+                z.extractall(site)
+        print('All wheels installed to ' + site)
+        PYEOF
+        else
+          # Fallback: try system pip (works if system has internet and compatible pip)
+          python3 -m pip install --break-system-packages flask lxml 2>/dev/null ||
+          pip3 install --break-system-packages flask lxml
+        fi
 
     - name: "Create systemd service for PXE webapp"
       copy:
@@ -392,7 +413,7 @@
           Environment=WEB_ROOT={{ web_root }}
           Environment=BLANCCO_REPORTS=/srv/samba/blancco-reports
           Environment=AUDIT_LOG=/var/log/pxe-webapp-audit.log
-          ExecStart=/opt/pxe-webapp/venv/bin/python app.py
+          ExecStart=/usr/bin/python3 app.py
           Restart=always
           RestartSec=5
 

test-vm.sh

@@ -4,17 +4,15 @@
 #
 # This script:
 #   1. Builds a CIDATA ISO with autoinstall config, packages, playbook, and webapp
-#   2. Creates an isolated libvirt network (pxe-test, 10.9.100.0/24)
+#   2. Launches an Ubuntu 24.04 Server VM on the default libvirt network
-#   3. Launches an Ubuntu 24.04 Server VM that auto-installs and configures itself
+#   3. The VM auto-installs, then runs the Ansible playbook on first boot
 #
 # Usage:
 #   ./test-vm.sh /path/to/ubuntu-24.04-live-server-amd64.iso
 #
-# After install completes (~10-15 min), access the webapp at:
+# After install completes (~10-15 min), access via:
-#   http://10.9.100.1:9009
+#   virsh console pxe-test       (serial console, always works)
-#
+#   ssh pxe@<dhcp-ip>            (check: virsh domifaddr pxe-test)
-# To watch progress:
-#   virsh console pxe-test
 #
 # To clean up:
 #   ./test-vm.sh --destroy
@@ -23,7 +21,6 @@ set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 VM_NAME="pxe-test"
-NET_NAME="pxe-test"
 VM_DISK="/var/lib/libvirt/images/${VM_NAME}.qcow2"
 CIDATA_ISO="/tmp/${VM_NAME}-cidata.iso"
 VM_RAM=4096
@@ -36,9 +33,8 @@ if [ "${1:-}" = "--destroy" ]; then
     virsh destroy "$VM_NAME" 2>/dev/null || true
     virsh undefine "$VM_NAME" 2>/dev/null || true
     rm -f "$VM_DISK"
-    virsh net-destroy "$NET_NAME" 2>/dev/null || true
-    virsh net-undefine "$NET_NAME" 2>/dev/null || true
     rm -f "$CIDATA_ISO"
+    rm -f "/tmp/${VM_NAME}-vmlinuz" "/tmp/${VM_NAME}-initrd"
     echo "Done."
     exit 0
 fi
@@ -94,6 +90,9 @@ fi
 if [ -d "$SCRIPT_DIR/pip-wheels" ]; then
     cp -r "$SCRIPT_DIR/pip-wheels" "$CIDATA_DIR/pip-wheels"
     echo "    Copied pip-wheels/"
+elif [ -d "$SCRIPT_DIR/offline-packages/pip-wheels" ]; then
+    cp -r "$SCRIPT_DIR/offline-packages/pip-wheels" "$CIDATA_DIR/pip-wheels"
+    echo "    Copied pip-wheels/ (from offline-packages/)"
 fi
 
 # Boot tools
@@ -108,39 +107,18 @@ CIDATA_SIZE=$(du -sh "$CIDATA_ISO" | cut -f1)
 echo "    CIDATA ISO: $CIDATA_ISO ($CIDATA_SIZE)"
 rm -rf "$CIDATA_DIR"
 
-# --- Step 2: Create isolated network ---
+# --- Step 2: Create VM disk ---
 echo ""
-echo "[2/4] Setting up isolated network ($NET_NAME)..."
+echo "[2/4] Creating VM disk (${VM_DISK_SIZE}GB)..."
-
-# Check if network already exists
-if virsh net-info "$NET_NAME" &>/dev/null; then
-    echo "    Network $NET_NAME already exists, reusing."
-else
-    cat > /tmp/${NET_NAME}-net.xml <<NETEOF
-<network>
-  <name>${NET_NAME}</name>
-  <bridge name="virbr-pxe" stp="on" delay="0"/>
-  <ip address="10.9.100.254" netmask="255.255.255.0"/>
-</network>
-NETEOF
-    virsh net-define /tmp/${NET_NAME}-net.xml
-    virsh net-start "$NET_NAME"
-    rm -f /tmp/${NET_NAME}-net.xml
-    echo "    Created isolated network 10.9.100.0/24 (no DHCP, no NAT)"
-fi
-
-# --- Step 3: Create VM disk ---
-echo ""
-echo "[3/4] Creating VM disk (${VM_DISK_SIZE}GB)..."
 if [ -f "$VM_DISK" ]; then
     echo "    Disk already exists. Destroy first with: $0 --destroy"
     exit 1
 fi
 qemu-img create -f qcow2 "$VM_DISK" "${VM_DISK_SIZE}G"
 
-# --- Step 4: Extract kernel/initrd from ISO ---
+# --- Step 3: Extract kernel/initrd from ISO ---
 echo ""
-echo "[4/5] Extracting kernel and initrd from ISO..."
+echo "[3/4] Extracting kernel and initrd from ISO..."
 ISO_MNT=$(mktemp -d)
 mount -o loop,ro "$UBUNTU_ISO" "$ISO_MNT"
 KERNEL="/tmp/${VM_NAME}-vmlinuz"
@@ -151,9 +129,13 @@ umount "$ISO_MNT"
 rmdir "$ISO_MNT"
 echo "    Extracted vmlinuz and initrd from casper/"
 
-# --- Step 5: Launch VM ---
+# --- Step 4: Launch VM ---
 echo ""
-echo "[5/5] Launching VM ($VM_NAME)..."
+echo "[4/4] Launching VM ($VM_NAME)..."
+
+# Use the default libvirt network (NAT, 192.168.122.0/24)
+# The VM gets a DHCP address initially for install access.
+# The Ansible playbook will later configure 10.9.100.1/24 for production use.
 virt-install \
     --name "$VM_NAME" \
     --memory "$VM_RAM" \
@@ -161,7 +143,7 @@ virt-install \
     --disk path="$VM_DISK",format=qcow2 \
     --disk path="$UBUNTU_ISO",device=cdrom,readonly=on \
     --disk path="$CIDATA_ISO",device=cdrom \
-    --network network="$NET_NAME" \
+    --network network=default \
     --os-variant ubuntu24.04 \
     --graphics none \
     --console pty,target_type=serial \
@@ -174,15 +156,20 @@ echo "VM launched! The autoinstall will take ~10-15 minutes."
 echo "============================================"
 echo ""
 echo "Watch progress:"
-echo "  virsh console $VM_NAME"
+echo "  sudo virsh console $VM_NAME"
 echo "  (Press Ctrl+] to detach)"
 echo ""
 echo "After install + first boot:"
-echo "  SSH:    ssh pxe@10.9.100.1"
+echo "  Console: sudo virsh console $VM_NAME"
-echo "  Webapp: http://10.9.100.1:9009"
+echo "  Find IP: sudo virsh domifaddr $VM_NAME"
+echo "  SSH:     ssh pxe@<ip-from-above>"
+echo ""
+echo "NOTE: The Ansible playbook will change the VM's IP to 10.9.100.1."
+echo "      After that, use 'virsh console' to access the VM."
+echo "      On the VM, verify with: curl http://localhost:9009"
 echo ""
 echo "Manage:"
-echo "  virsh start $VM_NAME"
+echo "  sudo virsh start $VM_NAME"
-echo "  virsh shutdown $VM_NAME"
+echo "  sudo virsh shutdown $VM_NAME"
 echo "  $0 --destroy   (remove everything)"
 echo ""