diff --git a/playbook/FlatUnattendW10-shopfloor.xml b/playbook/FlatUnattendW10-shopfloor.xml index ab010d3..cc8c03f 100644 --- a/playbook/FlatUnattendW10-shopfloor.xml +++ b/playbook/FlatUnattendW10-shopfloor.xml @@ -202,8 +202,8 @@ 4 - powershell.exe -ep Bypass -Command "net use Z: \\172.16.9.1\enrollment /user:pxe-upload pxe; robocopy Z:\shopfloor-setup C:\Enrollment Fetch-StagingPayload.ps1 Verify-And-Heal-Staging.ps1 /R:2 /W:2; robocopy Z:\scripts C:\Enrollment run-enrollment.ps1 wait-for-internet.ps1 migrate-to-wifi.ps1 /R:2 /W:2" - Bootstrap self-heal: pull Fetch/Verify-Heal + enrollment scripts from the PXE share so Order 5/6 run after an early WinPE staging failure. + powershell.exe -ep Bypass -Command "$u='pxe-upload'; $p='pxe'; $f='C:\Enrollment\fetch-source.txt'; if (Test-Path $f) { $l=@(Get-Content $f); if ($l.Count -ge 3) { $u=$l[1].Trim(); $p=$l[2].Trim() } }; net use Z: \\172.16.9.1\enrollment /user:$u $p; robocopy Z:\shopfloor-setup C:\Enrollment Fetch-StagingPayload.ps1 Verify-And-Heal-Staging.ps1 /R:2 /W:2; robocopy Z:\scripts C:\Enrollment run-enrollment.ps1 wait-for-internet.ps1 migrate-to-wifi.ps1 /R:2 /W:2" + Bootstrap self-heal: pull Fetch/Verify-Heal + enrollment scripts from the PXE share so Order 5/6 run after an early WinPE staging failure. Share login from fetch-source.txt (startnet writes it). 5 diff --git a/playbook/pxe_server_setup.yml b/playbook/pxe_server_setup.yml index e4c4901..1d074df 100644 --- a/playbook/pxe_server_setup.yml +++ b/playbook/pxe_server_setup.yml @@ -199,6 +199,17 @@ # letters, digits . _ ~ - (it sits in a URI, a kernel command line and # a mount option list), 8 to 64 of them. clonezilla_smb_password: "clonezilla-backup" + # The two other Samba accounts. pxe-upload: WinPE and Shop Floor setup + # on every imaging PC, image and app uploads. blancco: Blancco's report + # share. Same rules as clonezilla_smb_password. Empty: the shipped + # default (pxe / blancco). All three only SEED the webapp's Tools > + # Share accounts on the first run that sees them; after that the webapp + # owns them and a re-run keeps what it holds. + pxe_upload_password: "" + blancco_smb_password: "" + # The server's Linux login (pxe: SSH and sudo). Required while the box + # still has the shipped password "pxe"; set on every run it differs. + pxe_admin_password: "" # The site, as stamped on Blancco erasure reports and the monthly CMMC # evidence packs (webapp BLANCCO_SITE). Empty: "Unnamed site", so a # missing name shows instead of another site's. Letters, digits, spaces @@ -377,6 +388,46 @@ site-specific value in playbook/site-vars.yml and re-run. when: clonezilla_smb_password == "clonezilla-backup" + - name: "Shares: check the pxe-upload and blancco Samba passwords" + assert: + that: + - pxe_upload_password is string and blancco_smb_password is string + - pxe_upload_password | length == 0 or pxe_upload_password is match('^[A-Za-z0-9._~-]{8,64}$') + - blancco_smb_password | length == 0 or blancco_smb_password is match('^[A-Za-z0-9._~-]{8,64}$') + fail_msg: >- + pxe_upload_password and blancco_smb_password must be empty or 8-64 + characters from A-Z a-z 0-9 . _ ~ - (they go into startnet.cmd and + the Blancco preferences). Fix them in playbook/site-vars.yml and re-run. + quiet: yes + + - name: "Admin: is the Linux pxe login still the shipped password?" + shell: | + h=$(getent shadow pxe | cut -d: -f2) + case "$h" in + '$6$'*) salt=$(printf '%s' "$h" | cut -d'$' -f3) + if [ "$(printf 'pxe' | openssl passwd -6 -salt "$salt" -stdin)" = "$h" ]; then echo DEFAULT; else echo SITE; fi ;; + *) echo UNKNOWN ;; + esac + args: + executable: /bin/bash + register: pxe_admin_state + changed_when: false + + - name: "Admin: check pxe_admin_password" + assert: + that: + - pxe_admin_password is string + - pxe_admin_password | length > 0 or pxe_admin_state.stdout != 'DEFAULT' + - pxe_admin_password | length == 0 or (pxe_admin_password | length >= 8 and pxe_admin_password | length <= 128) + - pxe_admin_password | length == 0 or pxe_admin_password is not search('[\x00-\x1f]') + - pxe_admin_password != 'pxe' + fail_msg: >- + Set pxe_admin_password in playbook/site-vars.yml (8-128 characters, + no line breaks): the Linux login pxe (SSH and sudo) still has the + shipped password "pxe", the same on every server. Fix the file and + re-run. + quiet: yes + # --- Re-run safety (nothing below changes the box until these pass) --- - name: "Re-run safety: is imaging in progress? (SMB sessions, recent DHCP leases)" args: @@ -1473,28 +1524,49 @@ valid users = pxe-upload comment = Dell BIOS update bundle (read-only, served to WinPE) - - name: "Create Samba users (pxe-upload and blancco)" + # The Samba passwords belong to the webapp (Tools > Share accounts): + # here the accounts are only created, with the site-vars value or the + # default; "Shares: apply the stored passwords" near the end sets them. + - name: "Create Samba users (pxe-upload, blancco, clonezilla)" + # clonezilla: system user, no home, no login shell; only [clonezilla] + # lists it in valid users. shell: | + add() { # user password + pdbedit -L 2>/dev/null | grep -q "^$1:" && return 0 + printf '%s\n%s\n' "$2" "$2" | smbpasswd -a -s "$1" >/dev/null && echo "created $1" + } id pxe-upload >/dev/null 2>&1 || useradd -M -s /usr/sbin/nologin pxe-upload - echo -e 'pxe\npxe' | smbpasswd -a pxe-upload -s id blancco >/dev/null 2>&1 || useradd -M -s /usr/sbin/nologin blancco - echo -e 'blancco\nblancco' | smbpasswd -a blancco -s - args: - executable: /bin/bash - changed_when: false - - - name: "Create the clonezilla Samba user ([clonezilla] share only)" - # System user, no home, no login shell. Only [clonezilla] lists it in - # valid users. smbpasswd runs every time so a changed - # clonezilla_smb_password takes effect (the menu is updated with it). - shell: | id clonezilla >/dev/null 2>&1 || useradd -r -M -d /nonexistent -s /usr/sbin/nologin clonezilla - printf '%s\n%s\n' "$CZ_PW" "$CZ_PW" | smbpasswd -a -s clonezilla >/dev/null + add pxe-upload "${UP_PW:-pxe}" + add blancco "${BL_PW:-blancco}" + add clonezilla "$CZ_PW" environment: + UP_PW: "{{ pxe_upload_password }}" + BL_PW: "{{ blancco_smb_password }}" CZ_PW: "{{ clonezilla_smb_password }}" args: executable: /bin/bash - changed_when: false + register: smb_users + changed_when: "'created' in smb_users.stdout" + no_log: true + + - name: "Admin: set the Linux pxe password from pxe_admin_password" + shell: | + h=$(getent shadow pxe | cut -d: -f2) + salt=$(printf '%s' "$h" | cut -d'$' -f3) + if [ -n "$salt" ] && [ "$(printf '%s' "$PW" | openssl passwd -6 -salt "$salt" -stdin)" = "$h" ]; then + echo UNCHANGED + else + printf 'pxe:%s\n' "$PW" | chpasswd && echo CHANGED + fi + environment: + PW: "{{ pxe_admin_password }}" + args: + executable: /bin/bash + register: pxe_admin_set + changed_when: "'CHANGED' in pxe_admin_set.stdout" + when: pxe_admin_password | length > 0 no_log: true - name: "Deploy nf_conntrack TCP timeout sysctl for PXE workload" @@ -2001,12 +2073,6 @@ dest: "{{ web_root }}/blancco/blancco-progress.py" mode: '0644' - - name: "Ensure Samba user for Blancco reports exists (idempotent)" - shell: | - id blancco >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin blancco - pdbedit -L 2>/dev/null | grep -q '^blancco:' || (echo -e "blancco\nblancco" | smbpasswd -a -s blancco) - changed_when: false - - name: "Check for WinPE deployment content on USB" stat: path: "{{ usb_root }}/images" @@ -2422,6 +2488,28 @@ debug: msg: "{{ blancco_render.stdout_lines | select('search', 'CHANGED|WARNING|seeded') | list }}" + # The Samba passwords: seeded from site-vars only for accounts the + # webapp does not hold yet (Tools > Share accounts owns them after + # that), then set in Samba, startnet.cmd in boot.wim (this run injected + # the USB copy), the iPXE menu's Clonezilla entries (rendered from + # site-vars above) and the Blancco preferences. Passwords via env. + - name: "Shares: apply the stored passwords" + command: python3 -m services.share_accounts apply --seed + args: + chdir: /opt/pxe-webapp + environment: + WEB_ROOT: "{{ web_root }}" + SHARE_SEED_PXE_UPLOAD: "{{ pxe_upload_password }}" + SHARE_SEED_BLANCCO: "{{ blancco_smb_password }}" + SHARE_SEED_CLONEZILLA: "{{ clonezilla_smb_password }}" + no_log: true + register: share_apply + changed_when: share_apply.stdout_lines | select('search', 'updated|seeded') | list | length > 0 + + - name: "Shares: apply result" + debug: + msg: "{{ share_apply.stdout_lines }}" + # Hard gate: blancco-init.sh in initramfs drops to shell if the served # preferences.xml is malformed or missing the SMB target. Fail the # playbook here too so the problem surfaces at deploy time, not at the diff --git a/playbook/shopfloor-setup/Shopfloor/Invoke-PostPpkg.ps1 b/playbook/shopfloor-setup/Shopfloor/Invoke-PostPpkg.ps1 index e133b4a..bf2593b 100644 --- a/playbook/shopfloor-setup/Shopfloor/Invoke-PostPpkg.ps1 +++ b/playbook/shopfloor-setup/Shopfloor/Invoke-PostPpkg.ps1 @@ -13,6 +13,8 @@ # same, but it runs as Users and the HKLM write can be denied. # 3. Once Monitor-IntuneProgress has written C:\Enrollment\sync-complete.txt, # remove the "Shopfloor Intune Sync" logon task. +# Also, once the PPKG is applied, delete C:\Enrollment\fetch-source.txt (the +# pxe-upload share login startnet leaves for the first-logon scripts). # When all three are done the task unregisters itself. # # Never throws: every step logs and the script exits 0. @@ -47,6 +49,19 @@ if (Test-Path -LiteralPath $doneMarker) { } } +# 1b. fetch-source.txt holds the pxe-upload share password (startnet writes +# it for the first-logon Fetch/Verify scripts). Once the PPKG has run the +# PC is off the imaging LAN and the file has no use left. +$fetchSource = 'C:\Enrollment\fetch-source.txt' +if ((Test-Path -LiteralPath $appliedMarker) -and (Test-Path -LiteralPath $fetchSource)) { + try { + Remove-Item -LiteralPath $fetchSource -Force -ErrorAction Stop + Write-Host "Removed $fetchSource (share login)." + } catch { + Write-Warning "Could not remove ${fetchSource}: $_" + } +} + # 2. AutoLogonCount after the ShopFloor handover $handedOver = $false $countGone = $true diff --git a/scripts/Upload-Image.ps1 b/scripts/Upload-Image.ps1 index 7cb70d2..7bd2fd0 100644 --- a/scripts/Upload-Image.ps1 +++ b/scripts/Upload-Image.ps1 @@ -1,640 +1,647 @@ -# -# Upload-Image.ps1 - Copy the Media Creator Lite cache to the PXE server, -# then PROVE every file arrived intact. -# -# 1. Checks the MCL cache before sending anything: -# - exactly one -Production.tag at the cache root (MCL writes it only -# when a run finishes; none = an unfinished or interrupted cache) -# - no Control file newer than that tag (MCL still writing / crashed) -# - the OS in OperatingSystem.json is the one in user_selections.json -# - every Control / Tools JSON is non-empty and parses -# - every WIM, update and driver pack hashes to the SHA-256 MCL's own -# Control JSONs publish (MCL resumes downloads onto old files and then -# trusts any file of the right size, so a cache can hold junk) -# 2. Hashes every file it will send into upload-manifest.json. -# 3. Copies with robocopy as before (SMB share \\\image-upload). -# 4. POSTs the manifest to the PXE webapp, which hashes the files on the -# server's own disk (no Windows SMB cache in the way) and lists any file -# that is missing or differs. -# 5. Re-sends those files over HTTP (small files; the server writes a temp -# file, checks the hash, then renames it) or robocopy /J /IS (large -# files), and verifies again, up to -MaxRounds times. -# The webapp's Image Import only accepts an upload that passed step 4. -# -# Usage: -# .\Upload-Image.ps1 (selected OS + packages, no drivers) -# .\Upload-Image.ps1 -IncludeDrivers (also upload selected hardware drivers) -# .\Upload-Image.ps1 -CachePath "D:\MCL\Cache" (custom cache location) -# .\Upload-Image.ps1 -Server 172.16.9.1 (custom server IP) -# -# Exit codes: 0 verified, 1 failed (details printed), 2 MCL cache not usable. -# - -param( - [string]$CachePath = "C:\ProgramData\GEAerospace\MediaCreator\Cache", - [string]$Server = "172.16.9.1", - [int]$WebPort = 9009, - [string]$User = "pxe-upload", - [string]$Pass = "pxe", - [int]$MaxRounds = 3, - [switch]$IncludeDrivers -) - -$ErrorActionPreference = "Stop" -$Share = "\\$Server\image-upload" -$ApiBase = "http://${Server}:$WebPort" -$ExcludedDeployDirs = @("Operating Systems", "Out-of-box Drivers", "Packages") - -function Format-Size { - param([long]$Bytes) - if ($Bytes -ge 1GB) { return "{0:N1} GB" -f ($Bytes / 1GB) } - if ($Bytes -ge 1MB) { return "{0:N1} MB" -f ($Bytes / 1MB) } - return "{0:N0} KB" -f ($Bytes / 1KB) -} - -function Resolve-DestDir { - param([string]$Dir) - $d = $Dir -replace '^\*destinationdir\*\\?', '' - return (($d -replace '^[A-Za-z]:\\', '').Trim('\')) -} - -function Write-Step { - param([string]$Text) - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host $Text -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan -} - -function Stop-Cache { - # The cache itself is unusable: nothing was sent. - param([string[]]$Lines) - Write-Host "" - foreach ($l in $Lines) { Write-Host "ERROR: $l" -ForegroundColor Red } - Write-Host "Nothing was uploaded." -ForegroundColor Red - exit 2 -} - -function Read-JsonFile { - param([string]$Path) - return (Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json) -} - -# HTTP to the webapp with HttpWebRequest and NO proxy: the work PC's system -# proxy (Zscaler) must not see traffic to the isolated imaging LAN. -# Returns @{ Status = ; Body = }; never throws on HTTP errors. -function Invoke-Pxe { - param([string]$Method, [string]$Path, [byte[]]$Body, [string]$InFile, - [string]$ContentType = "application/json", [int]$TimeoutSec = 120) - $req = [System.Net.HttpWebRequest]::Create("$ApiBase$Path") - $req.Method = $Method - $req.Proxy = $null - $req.Timeout = $TimeoutSec * 1000 - $req.ReadWriteTimeout = $TimeoutSec * 1000 - $req.KeepAlive = $false - if ($Body -or $InFile) { - $req.ContentType = $ContentType - if ($InFile) { $req.ContentLength = (Get-Item -LiteralPath $InFile).Length } - else { $req.ContentLength = $Body.Length } - try { $rs = $req.GetRequestStream() } catch { return @{ Status = 0; Body = $_.Exception.Message } } - try { - if ($InFile) { - $fs = [System.IO.File]::OpenRead($InFile) - try { $fs.CopyTo($rs) } finally { $fs.Dispose() } - } else { - $rs.Write($Body, 0, $Body.Length) - } - } finally { $rs.Dispose() } - } - try { - $resp = $req.GetResponse() - } catch { - # A .NET method call wraps the WebException in a - # MethodInvocationException: look at the inner one. - $ex = $_.Exception - while ($ex -and -not ($ex -is [System.Net.WebException])) { $ex = $ex.InnerException } - if (-not $ex -or -not $ex.Response) { return @{ Status = 0; Body = $_.Exception.Message } } - $resp = $ex.Response - } - try { - $sr = New-Object System.IO.StreamReader($resp.GetResponseStream()) - $text = $sr.ReadToEnd() - $sr.Dispose() - return @{ Status = [int]$resp.StatusCode; Body = $text } - } finally { $resp.Close() } -} - -function Get-ErrorText { - param($Resp) - try { return ($Resp.Body | ConvertFrom-Json).error } catch { return $Resp.Body } -} - -Write-Host "" -Write-Host "========================================" -ForegroundColor Cyan -Write-Host " PXE Server Image Uploader" -ForegroundColor Cyan -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "" - -# --- Validate source paths --- -$CachePath = $CachePath.TrimEnd('\') -$DeployPath = Join-Path $CachePath "Deploy" -$ToolsPath = Join-Path (Split-Path $CachePath -Parent) "Tools" -if (-not (Test-Path $ToolsPath -PathType Container)) { - $ToolsPath = "C:\ProgramData\GEAerospace\MediaCreator\Tools" -} -$SourcesZip = Join-Path $CachePath "Boot\Sources.zip" -$ControlPath = Join-Path $DeployPath "Control" - -if (-not (Test-Path $DeployPath -PathType Container)) { - Write-Host "ERROR: Deploy directory not found at $DeployPath" -ForegroundColor Red - Write-Host " .\Upload-Image.ps1 -CachePath ""D:\Path\To\Cache""" -ForegroundColor Yellow - exit 2 -} - -# --- MCL finished? One tag at the cache root, written at the very end --- -$tags = @(Get-ChildItem -LiteralPath $CachePath -Filter "*.tag" -File -ErrorAction SilentlyContinue) -if ($tags.Count -eq 0) { - Stop-Cache @("No MCL completion tag (*.tag) in $CachePath.", - "MCL writes it only when a run finishes: the cache is incomplete or the run was interrupted.", - "Run Media Creator Lite again and let it finish.") -} -if ($tags.Count -gt 1) { - Stop-Cache (@("More than one MCL tag in ${CachePath}:") + ($tags | ForEach-Object { " $($_.Name)" }) + - @("MCL leaves exactly one. Run Media Creator Lite again.")) -} -$tag = $tags[0] -$tagBytes = [System.IO.File]::ReadAllBytes($tag.FullName) -if ($tagBytes.Length -lt 8) { Stop-Cache @("MCL tag $($tag.Name) is damaged ($($tagBytes.Length) bytes). Run MCL again.") } -# MCL writes DateTime.Now.Ticks (local time) into the tag (TagFactory). -try { $tagTime = New-Object DateTime ([BitConverter]::ToInt64($tagBytes, 0)) } -catch { Stop-Cache @("MCL tag $($tag.Name) holds no valid time. Run MCL again.") } -$tagOs = $tag.BaseName -replace '-(Production|QA)$', '' - -$late = @(Get-ChildItem -LiteralPath $ControlPath -File -ErrorAction SilentlyContinue | - Where-Object { $_.LastWriteTime -gt $tagTime.AddSeconds(5) }) -if ($late.Count -gt 0) { - Stop-Cache (@("These Control files are newer than the MCL finish tag ($tagTime):") + - ($late | ForEach-Object { " $($_.Name) $($_.LastWriteTime)" }) + - @("MCL was still writing or crashed. Run Media Creator Lite again and let it finish.")) -} - -# --- Every Control / Tools JSON must be non-empty and parse --- -$badJson = @() -foreach ($dir in @($ControlPath, $ToolsPath)) { - if (-not (Test-Path $dir -PathType Container)) { continue } - foreach ($j in Get-ChildItem -LiteralPath $dir -Filter "*.json" -File -Recurse) { - if ($j.Length -eq 0) { $badJson += "$($j.FullName) (0 bytes)"; continue } - try { $null = Read-JsonFile $j.FullName } catch { $badJson += "$($j.FullName) (not valid JSON)" } - } -} -if ($badJson.Count -gt 0) { - Stop-Cache (@("Broken JSON in the MCL cache:") + ($badJson | ForEach-Object { " $_" }) + - @("Delete the cache folder ($CachePath) and run Media Creator Lite again.")) -} - -# --- Parse user_selections.json --- -$SelectionsFile = Join-Path $ToolsPath "user_selections.json" -if (-not (Test-Path $SelectionsFile)) { - Write-Host "ERROR: user_selections.json not found at $SelectionsFile" -ForegroundColor Red - Write-Host " Run Media Creator Lite first to create a configuration." -ForegroundColor Yellow - exit 2 -} - -$selections = (Read-JsonFile $SelectionsFile)[0] -$selectedOsId = $selections.OperatingSystemSelection -$selectedModelIds = @($selections.HardwareModelSelection | ForEach-Object { $_.Id } | Select-Object -Unique) - -# --- Parse control JSONs --- -$osJsonFile = Join-Path $ControlPath "OperatingSystem.json" -$driverJsonFile = Join-Path $ControlPath "HardwareDriver.json" -$pkgJsonFile = Join-Path $ControlPath "packages.json" -$langJsonFile = Join-Path $ControlPath "LanguagePacks.json" - -if (-not (Test-Path $osJsonFile)) { - Write-Host "ERROR: OperatingSystem.json not found in $ControlPath" -ForegroundColor Red - exit 2 -} - -$osJson = @(Read-JsonFile $osJsonFile) -$driverJson = if (Test-Path $driverJsonFile) { @(Read-JsonFile $driverJsonFile) } else { @() } -$pkgJson = if (Test-Path $pkgJsonFile) { @(Read-JsonFile $pkgJsonFile) } else { @() } -$langJson = if (Test-Path $langJsonFile) { @(Read-JsonFile $langJsonFile) } else { @() } - -# --- The cached OS must be the selected one, and the one the tag names --- -$matchedOs = @($osJson | Where-Object { $_.operatingSystemVersion.id -eq [int]$selectedOsId }) -if ($matchedOs.Count -eq 0) { - Stop-Cache @("OperatingSystem.json does not hold the OS selected in user_selections.json (ID $selectedOsId).", - "Run Media Creator Lite again.") -} -$osv = $matchedOs[0].operatingSystemVersion -if (-not $tagOs.StartsWith($osv.productName, [StringComparison]::OrdinalIgnoreCase)) { - Stop-Cache @("The MCL tag says '$tagOs' but OperatingSystem.json holds '$($osv.productName)'.", - "The cache is a mix of two runs. Run Media Creator Lite again.") -} -$isLtsc = $osv.productName -match 'LTSC' -$suggested = if ($isLtsc) { "gea-shopfloor" } else { "standard-or-engineer" } - -# --- Resolve selections to paths --- -$osDirs = @() -$osTotalSize = [long]0 -foreach ($os in $matchedOs) { - $osDirs += Resolve-DestDir $os.operatingSystemVersion.wim.DestinationDir - $osTotalSize += $os.operatingSystemVersion.wim.size -} - -# Packages: enabled + matching OS ID (osId is a comma string in the new schema) -function Get-OsIds { param($e) if ($e.aOsIds) { return @($e.aOsIds) } return @("$($e.osId)" -split ',' | ForEach-Object { $_.Trim() }) } -$matchedPkgs = @($pkgJson | Where-Object { (Get-OsIds $_) -contains "$selectedOsId" -and $_.enabled -eq 1 }) -$pkgTotalSize = [long]0 -foreach ($pkg in $matchedPkgs) { $pkgTotalSize += $pkg.size } - -# Drivers: match selected model IDs (family) + OS ID, deduplicate by path -$allMatchingDrivers = @($driverJson | Where-Object { - $selectedModelIds -contains $_.family -and (Get-OsIds $_) -contains "$selectedOsId" -}) -$allDriverDirSet = [ordered]@{} -foreach ($drv in $allMatchingDrivers) { - $rel = Resolve-DestDir $drv.DestinationDir - if (-not $allDriverDirSet.Contains($rel)) { $allDriverDirSet[$rel] = $drv.size } -} -$allDriverCount = $allDriverDirSet.Count -$allDriverTotalSize = [long]0 -$allDriverDirSet.Values | ForEach-Object { $allDriverTotalSize += $_ } - -$driverDirs = @() -$driverTotalSize = [long]0 -if ($IncludeDrivers) { - $driverDirs = @($allDriverDirSet.Keys) - $driverTotalSize = $allDriverTotalSize -} - -# --- Display upload plan --- -Write-Host " Cache: $CachePath" -Write-Host " MCL run: $tagOs, finished $tagTime" -ForegroundColor Green -Write-Host " Server: $Server" -Write-Host " Target: $(if ($isLtsc) { 'Shop Floor (LTSC)' } else { 'Standard or Engineer (pick it in the webapp)' })" -Write-Host "" -Write-Host " Upload Plan (from user_selections.json):" -ForegroundColor Cyan -Write-Host " ------------------------------------------" -Write-Host " OS: $($osv.marketingName) $($osv.productName) ($(Format-Size $osTotalSize))" -ForegroundColor Green -Write-Host " Packages: $($matchedPkgs.Count) update(s) ($(Format-Size $pkgTotalSize))" -ForegroundColor Green -if ($IncludeDrivers) { - Write-Host " Drivers: $($driverDirs.Count) model(s) ($(Format-Size $driverTotalSize))" -ForegroundColor Green -} else { - Write-Host " Drivers: SKIPPED -- $allDriverCount available, use -IncludeDrivers" -ForegroundColor Yellow -} -Write-Host " Control: Always included" -ForegroundColor Gray -Write-Host " Tools: $(if (Test-Path $ToolsPath) { 'Yes' } else { 'Not found' })" -ForegroundColor $(if (Test-Path $ToolsPath) { "Gray" } else { "Yellow" }) -Write-Host " Sources: $(if (Test-Path $SourcesZip) { 'Yes (from Boot\Sources.zip)' } else { 'Not found' })" -ForegroundColor $(if (Test-Path $SourcesZip) { "Gray" } else { "Yellow" }) -Write-Host "" - -# --- Sources: extract now so it can be hashed with everything else --- -$TempExtract = $null -$TempSources = $null -if (Test-Path $SourcesZip) { - $TempExtract = Join-Path $env:TEMP "SourcesExtract" - Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue - Expand-Archive $SourcesZip -DestinationPath $TempExtract -Force - # Handle nested Sources folder (zip may contain Sources/ at root) - $TempSources = $TempExtract - if ((Test-Path (Join-Path $TempExtract "Sources")) -and -not (Test-Path (Join-Path $TempExtract "Diskpart"))) { - $TempSources = Join-Path $TempExtract "Sources" - } -} - -# --- Everything that will be sent: local file + path on the share --- -$sendList = New-Object System.Collections.Generic.List[object] -function Add-Tree { - param([string]$LocalDir, [string]$RelPrefix, [string[]]$SkipDirs = @()) - if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { return } - $base = (Get-Item -LiteralPath $LocalDir).FullName.TrimEnd('\') - foreach ($f in Get-ChildItem -LiteralPath $base -Recurse -File -Force) { - $rel = $f.FullName.Substring($base.Length + 1) - if ($SkipDirs.Count -gt 0) { - # robocopy /XD skips that folder name at any depth - $segs = $rel.Split('\') - $skip = $false - for ($i = 0; $i -lt $segs.Count - 1; $i++) { if ($SkipDirs -contains $segs[$i]) { $skip = $true; break } } - if ($skip) { continue } - } - $sendList.Add([pscustomobject]@{ Local = $f.FullName; Rel = (Join-Path $RelPrefix $rel); Size = $f.Length }) - } -} -Add-Tree $DeployPath "Deploy" $ExcludedDeployDirs -foreach ($d in $osDirs) { Add-Tree (Join-Path $CachePath $d) $d } -foreach ($pkg in $matchedPkgs) { - $pdir = Resolve-DestDir $pkg.destinationDir - $rel = if ($pdir) { Join-Path $pdir $pkg.fileName } else { $pkg.fileName } - $local = Join-Path $CachePath $rel - if (Test-Path -LiteralPath $local -PathType Leaf) { - # File name as robocopy will write it (disk case), not as the JSON spells it - $item = Get-Item -LiteralPath $local - $rel = Join-Path (Split-Path $rel -Parent) $item.Name - $sendList.Add([pscustomobject]@{ Local = $item.FullName; Rel = $rel; Size = $item.Length }) - } -} -foreach ($d in $driverDirs) { Add-Tree (Join-Path $CachePath $d) $d } -Add-Tree $ToolsPath "Tools" -if ($TempSources) { Add-Tree $TempSources "Sources" } - -# The same file can be reached twice (an OS folder outside the excluded -# Deploy dirs, nested driver folders, two packages naming one file); a -# duplicate path would make the server refuse the whole manifest. -$seenRel = @{} -$dedup = New-Object System.Collections.Generic.List[object] -foreach ($f in $sendList) { - $k = ($f.Rel -replace '\\', '/').ToLower() - if (-not $seenRel.ContainsKey($k)) { $seenRel[$k] = $true; $dedup.Add($f) } -} -$sendList = $dedup - -# --- Hash everything; check payload against MCL's published hashes --- -Write-Step "Hashing $($sendList.Count) file(s) ($(Format-Size (($sendList | Measure-Object Size -Sum).Sum)))" -$mclHash = @{} -function Add-Mcl { param($Dest, $Name, $Hash) - # Only a real SHA-256 (as the server checks): anything else is not ours to judge - if (-not ($Dest -and $Name -and ("$Hash" -match '^[0-9a-fA-F]{64}$'))) { return } - $dd = Resolve-DestDir $Dest - $key = if ($dd) { Join-Path $dd $Name } else { "$Name" } - $mclHash[$key.ToLower()] = "$Hash".ToLower() } -foreach ($e in $osJson) { - $w = $e.operatingSystemVersion.wim - $n = if ($w.fileName) { $w.fileName } else { "install.wim" } - Add-Mcl $w.destinationDir $n $w.hash -} -foreach ($e in @($driverJson) + @($pkgJson)) { if ($e) { Add-Mcl $e.destinationDir $e.fileName $e.hash } } -foreach ($e in $langJson) { foreach ($lf in @($e.files)) { if ($lf) { Add-Mcl $lf.destFolder $lf.fileName $lf.hash } } } - -$totalBytes = [long](($sendList | Measure-Object Size -Sum).Sum) -$doneBytes = [long]0 -$badCache = @() -$i = 0 -foreach ($f in $sendList) { - $i++ - $pct = if ($totalBytes -gt 0) { [int](100 * $doneBytes / $totalBytes) } else { 100 } - Write-Progress -Activity "Hashing" -Status "$i/$($sendList.Count) $($f.Rel)" -PercentComplete $pct - $f | Add-Member -NotePropertyName Sha256 -NotePropertyValue (Get-FileHash -LiteralPath $f.Local -Algorithm SHA256).Hash.ToLower() - $doneBytes += $f.Size - $want = $mclHash[$f.Rel.ToLower()] - if ($want -and $want -ne $f.Sha256) { $badCache += $f } -} -Write-Progress -Activity "Hashing" -Completed -if ($badCache.Count -gt 0) { - Write-Host "" - Write-Host "These cache files do not match the hash MCL publishes for them (damaged download):" -ForegroundColor Red - foreach ($b in $badCache) { Write-Host " $($b.Rel)" -ForegroundColor Red } - Write-Host "" - Write-Host "MCL re-downloads a MISSING file correctly, but trusts a damaged one of the right size." -ForegroundColor Yellow - $ans = Read-Host "Delete these $($badCache.Count) file(s) from the MCL cache so MCL downloads them again? (Y/N)" - if ($ans -match '^[Yy]') { - foreach ($b in $badCache) { Remove-Item -LiteralPath $b.Local -Force; Write-Host " deleted $($b.Local)" -ForegroundColor Gray } - Write-Host "Now run Media Creator Lite again, then this script." -ForegroundColor Yellow - } - if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } - Write-Host "Nothing was uploaded." -ForegroundColor Red - exit 2 -} -Write-Host " All $($sendList.Count) file(s) hashed; payload matches MCL's published hashes." -ForegroundColor Green - -$pcName = ($env:COMPUTERNAME -replace '[^A-Za-z0-9._-]', '') -$uploadId = "{0}-{1}" -f (Get-Date -Format "yyyyMMdd-HHmmss"), $pcName -if ($uploadId.Length -gt 80) { $uploadId = $uploadId.Substring(0, 80) } -$manifest = [ordered]@{ - manifest_version = 1 - upload_id = $uploadId - created = (Get-Date -Format "yyyy-MM-ddTHH:mm:sszzz") - source_pc = $pcName - mcl = [ordered]@{ - tag = $tag.Name - finished = $tagTime.ToString("yyyy-MM-ddTHH:mm:ss") - os_id = [int]$osv.id - os_name = "$($osv.productName)" - os_version = "$($osv.versionNumber)" - build = "$($osv.buildNumber)" - suggested_target = $suggested - } - files = @($sendList | ForEach-Object { [ordered]@{ path = ($_.Rel -replace '\\', '/'); size = [long]$_.Size; sha256 = $_.Sha256 } }) -} -$manifestBytes = [System.Text.Encoding]::UTF8.GetBytes(($manifest | ConvertTo-Json -Depth 6 -Compress)) - -# --- Is the webapp reachable? (before spending time on the copy) --- -$ping = Invoke-Pxe -Method GET -Path "/api/upload/verify" -TimeoutSec 15 -if ($ping.Status -ne 200) { - Write-Host "ERROR: the PXE webapp at $ApiBase does not answer ($($ping.Status) $($ping.Body))." -ForegroundColor Red - Write-Host " The upload cannot be verified without it. Check the server, then run this again." -ForegroundColor Yellow - if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } - exit 1 -} - -# --- Connect to SMB share --- -Write-Host "" -Write-Host "Connecting to $Share ..." -ForegroundColor Gray -# net use goes through cmd: in Windows PowerShell 5.1 a native command's -# redirected stderr becomes an error record, which ErrorAction Stop turns -# into a crash ("The network connection could not be found"). -cmd /c "net use ""$Share"" /delete /y >nul 2>&1" | Out-Null - -$netResult = cmd /c "net use ""$Share"" /user:$User ""$Pass"" 2>&1" -if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: Could not connect to $Share" -ForegroundColor Red - Write-Host $netResult -ForegroundColor Red - Write-Host "" - Write-Host "Make sure:" -ForegroundColor Yellow - Write-Host " - The PXE server is running at $Server" -ForegroundColor Yellow - Write-Host " - This PC is on the 172.16.9.x imaging network" -ForegroundColor Yellow - Write-Host " - Samba is running on the PXE server" -ForegroundColor Yellow - if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } - exit 1 -} -Write-Host "Connected." -ForegroundColor Green - -$failed = $false -$stepNum = 0 -$totalSteps = 2 # Deploy base + OS always -if ($matchedPkgs.Count -gt 0) { $totalSteps++ } -if ($IncludeDrivers -and $driverDirs.Count -gt 0) { $totalSteps++ } -if (Test-Path $ToolsPath -PathType Container) { $totalSteps++ } -if ($TempSources) { $totalSteps++ } - -# robocopy exit codes 8 and up are failures -$rcArgs = @("/R:3", "/W:5", "/NP", "/ETA") - -# --- Step: Deploy base (Control, Applications, config -- skip big dirs) --- -$stepNum++ -Write-Step "[$stepNum/$totalSteps] Copying Deploy\ base (Control, Applications, config) ..." -robocopy $DeployPath "$Share\Deploy" /E /XD @ExcludedDeployDirs @rcArgs -if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Deploy base copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } - -# --- Step: Operating System --- -$stepNum++ -Write-Step "[$stepNum/$totalSteps] Copying Operating System ..." -foreach ($osDir in $osDirs) { - $src = Join-Path $CachePath $osDir - if (Test-Path -LiteralPath $src -PathType Container) { - Write-Host " $osDir" -ForegroundColor Gray - robocopy $src (Join-Path $Share $osDir) /E @rcArgs - if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: OS copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } - } else { - Write-Host " SKIPPED (not cached): $osDir" -ForegroundColor Yellow - } -} - -# --- Step: Packages --- -if ($matchedPkgs.Count -gt 0) { - $stepNum++ - Write-Step "[$stepNum/$totalSteps] Copying Packages ..." - $pkgGroups = [ordered]@{} - foreach ($pkg in $matchedPkgs) { - $rel = Resolve-DestDir $pkg.destinationDir - if (-not $pkgGroups.Contains($rel)) { $pkgGroups[$rel] = @() } - $pkgGroups[$rel] += $pkg.fileName - } - foreach ($dir in $pkgGroups.Keys) { - $src = Join-Path $CachePath $dir - if (Test-Path -LiteralPath $src -PathType Container) { - foreach ($f in $pkgGroups[$dir]) { Write-Host " $f" -ForegroundColor Gray } - $robocopyArgs = @($src, (Join-Path $Share $dir)) + $pkgGroups[$dir] + $rcArgs - & robocopy @robocopyArgs - if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Package copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } - } else { - Write-Host " SKIPPED (not cached): $dir" -ForegroundColor Yellow - } - } -} - -# --- Step: Drivers (only with -IncludeDrivers) --- -if ($IncludeDrivers -and $driverDirs.Count -gt 0) { - $stepNum++ - Write-Step "[$stepNum/$totalSteps] Copying Drivers ($($driverDirs.Count) model(s)) ..." - $drvCopied = 0 - foreach ($drvDir in $driverDirs) { - $drvCopied++ - $src = Join-Path $CachePath $drvDir - if (Test-Path -LiteralPath $src -PathType Container) { - Write-Host " [$drvCopied/$($driverDirs.Count)] $drvDir" -ForegroundColor Gray - robocopy $src (Join-Path $Share $drvDir) /E @rcArgs - if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Driver copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } - } else { - Write-Host " SKIPPED (not cached): $drvDir" -ForegroundColor Yellow - } - } -} - -# --- Step: Tools --- -if (Test-Path $ToolsPath -PathType Container) { - $stepNum++ - Write-Step "[$stepNum/$totalSteps] Copying Tools\ ..." - robocopy $ToolsPath "$Share\Tools" /E @rcArgs - if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Tools copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } -} - -# --- Step: Sources --- -if ($TempSources) { - $stepNum++ - Write-Step "[$stepNum/$totalSteps] Copying Sources\ ..." - robocopy $TempSources "$Share\Sources" /E @rcArgs - if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Sources copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } -} - -# --- Verify on the server; re-send what is wrong --- -$byRel = @{} -foreach ($f in $sendList) { $byRel[($f.Rel -replace '\\', '/').ToLower()] = $f } -$verified = $false -$result = $null -for ($round = 1; $round -le $MaxRounds + 1; $round++) { - Write-Step "Verifying on the server (round $round) ..." - $r = Invoke-Pxe -Method POST -Path "/api/upload/verify" -Body $manifestBytes -TimeoutSec 120 - if ($r.Status -ne 202) { - Write-Host "ERROR: verify request refused ($($r.Status)): $(Get-ErrorText $r)" -ForegroundColor Red - break - } - $deadline = (Get-Date).AddHours(2) - $postedAt = Get-Date - do { - if ((Get-Date) -gt $deadline) { $result = $null; break } - if ($result -and $result.upload_id -ne $uploadId -and ((Get-Date) - $postedAt).TotalSeconds -gt 120) { - Write-Host "ERROR: the server is working on another upload ($($result.upload_id)). Run this again later." -ForegroundColor Red - $result = $null; break - } - Start-Sleep -Seconds 2 - $s = Invoke-Pxe -Method GET -Path "/api/upload/verify" -TimeoutSec 30 - $result = $null - if ($s.Status -eq 200) { $result = $s.Body | ConvertFrom-Json } - if ($result -and $result.state -eq "running" -and $result.total_bytes -gt 0) { - Write-Progress -Activity "Server is hashing the upload" -Status "$($result.done_files)/$($result.total_files) files" ` - -PercentComplete ([Math]::Min(100, [int](100 * $result.done_bytes / $result.total_bytes))) - } - } while (-not $result -or $result.state -eq "running" -or $result.upload_id -ne $uploadId) - Write-Progress -Activity "Server is hashing the upload" -Completed - - if (-not $result) { Write-Host "ERROR: the server did not finish verifying within 2 hours." -ForegroundColor Red; break } - if ($result.state -eq "error") { Write-Host "ERROR: the server could not verify: $($result.error)" -ForegroundColor Red; break } - if ($result.ok) { $verified = $true; break } - - foreach ($b in @($result.bad_in_cache)) { if ($b) { Write-Host " BAD IN MCL CACHE: $($b.path) ($($b.reason))" -ForegroundColor Red } } - if (@($result.bad_in_cache).Count -gt 0) { break } # re-sending a bad source cannot help - - $redo = @(@($result.missing) + @($result.mismatch | ForEach-Object { $_.path }) | Where-Object { $_ }) - Write-Host " $($redo.Count) file(s) missing or different on the server." -ForegroundColor Yellow - if ($round -gt $MaxRounds) { break } - foreach ($rel in $redo) { - $f = $byRel[$rel.ToLower()] - if (-not $f) { continue } - if ($f.Size -le [long]$result.small_max) { - $q = "?upload_id=$([uri]::EscapeDataString($uploadId))&path=$([uri]::EscapeDataString($rel))" - $p = Invoke-Pxe -Method PUT -Path "/api/upload/file$q" -InFile $f.Local -ContentType "application/octet-stream" -TimeoutSec 300 - if ($p.Status -eq 200) { Write-Host " re-sent over HTTP: $rel" -ForegroundColor Green } - else { Write-Host " HTTP re-send failed ($($p.Status)) for ${rel}: $(Get-ErrorText $p)" -ForegroundColor Red } - } else { - $dstDir = Join-Path $Share (Split-Path ($f.Rel) -Parent) - $dstFile = Join-Path $dstDir (Split-Path $f.Local -Leaf) - Write-Host " re-copying (delete, then robocopy /J): $rel" -ForegroundColor Gray - # robocopy calls a same-size, same-time file "same" or "modified" - # and skips it, even with /IS /IT. Delete the bad copy so it is a - # new file; /IS /IT /IM stay as a backstop if the delete fails. - Remove-Item -LiteralPath $dstFile -Force -ErrorAction SilentlyContinue - $rc = robocopy (Split-Path $f.Local -Parent) $dstDir (Split-Path $f.Local -Leaf) /J /IS /IT /IM @rcArgs - if ($LASTEXITCODE -ge 8 -or $LASTEXITCODE -eq 0) { - # 0 = nothing copied: robocopy skipped it again - Write-Host " robocopy did not copy $rel (exit $LASTEXITCODE):" -ForegroundColor Red - $rc | Where-Object { $_ -match '\S' } | Select-Object -Last 8 | ForEach-Object { Write-Host " $_" } - } - } - } -} - -# --- Disconnect and clean up --- -cmd /c "net use ""$Share"" /delete /y >nul 2>&1" | Out-Null -if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } - -# --- Summary --- -Write-Host "" -if ($failed -and $verified) { - Write-Host " (robocopy reported errors on the first pass; the files it missed were re-sent and verified.)" -ForegroundColor Yellow -} -if ($verified) { - Write-Host "========================================" -ForegroundColor Green - Write-Host " Upload complete: all $($sendList.Count) file(s) verified on the server." -ForegroundColor Green - Write-Host "========================================" -ForegroundColor Green - if ([int]$result.extra_count -gt 0) { - Write-Host "" - Write-Host " NOTE: the upload folder also holds $($result.extra_count) file(s) from an earlier upload." -ForegroundColor Yellow - Write-Host " Import is blocked until you remove them (button on the Image Import page)." -ForegroundColor Yellow - } - Write-Host "" - Write-Host "Next steps:" -ForegroundColor Cyan - Write-Host " 1. Open the PXE webapp: http://${Server}:$WebPort/images/import" -ForegroundColor White - Write-Host " 2. Target: $(if ($isLtsc) { 'Shop Floor (the only choice for LTSC)' } else { 'Standard or Engineer' })" -ForegroundColor White - Write-Host " 3. Click Start import" -ForegroundColor White - Write-Host "" - exit 0 -} -Write-Host "========================================" -ForegroundColor Red -Write-Host " Upload NOT verified." -ForegroundColor Red -Write-Host "========================================" -ForegroundColor Red -if ($result) { - foreach ($m in @($result.missing)) { if ($m) { Write-Host " missing: $m" -ForegroundColor Red } } - foreach ($m in @($result.mismatch)) { if ($m) { Write-Host " differs: $($m.path) ($($m.reason))" -ForegroundColor Red } } -} -Write-Host "The webapp will not import this upload. Run the script again." -ForegroundColor Yellow -exit 1 +# +# Upload-Image.ps1 - Copy the Media Creator Lite cache to the PXE server, +# then PROVE every file arrived intact. +# +# 1. Checks the MCL cache before sending anything: +# - exactly one -Production.tag at the cache root (MCL writes it only +# when a run finishes; none = an unfinished or interrupted cache) +# - no Control file newer than that tag (MCL still writing / crashed) +# - the OS in OperatingSystem.json is the one in user_selections.json +# - every Control / Tools JSON is non-empty and parses +# - every WIM, update and driver pack hashes to the SHA-256 MCL's own +# Control JSONs publish (MCL resumes downloads onto old files and then +# trusts any file of the right size, so a cache can hold junk) +# 2. Hashes every file it will send into upload-manifest.json. +# 3. Copies with robocopy as before (SMB share \\\image-upload). +# 4. POSTs the manifest to the PXE webapp, which hashes the files on the +# server's own disk (no Windows SMB cache in the way) and lists any file +# that is missing or differs. +# 5. Re-sends those files over HTTP (small files; the server writes a temp +# file, checks the hash, then renames it) or robocopy /J /IS (large +# files), and verifies again, up to -MaxRounds times. +# The webapp's Image Import only accepts an upload that passed step 4. +# +# Usage: +# .\Upload-Image.ps1 (selected OS + packages, no drivers) +# .\Upload-Image.ps1 -IncludeDrivers (also upload selected hardware drivers) +# .\Upload-Image.ps1 -CachePath "D:\MCL\Cache" (custom cache location) +# .\Upload-Image.ps1 -Server 172.16.9.1 (custom server IP) +# .\Upload-Image.ps1 -Pass (pxe-upload password; asked for if not given) +# +# Exit codes: 0 verified, 1 failed (details printed), 2 MCL cache not usable. +# + +param( + [string]$CachePath = "C:\ProgramData\GEAerospace\MediaCreator\Cache", + [string]$Server = "172.16.9.1", + [int]$WebPort = 9009, + [string]$User = "pxe-upload", + [string]$Pass = "", # empty: asked for (Tools > Share accounts sets it) + [int]$MaxRounds = 3, + [switch]$IncludeDrivers +) + +$ErrorActionPreference = "Stop" +$Share = "\\$Server\image-upload" +# Asked for up front, before the cache checks that can take a while. +if (-not $Pass) { + $sec = Read-Host -AsSecureString "Password of $User on $Server (Enter = the shipped default)" + $Pass = [Runtime.InteropServices.Marshal]::PtrToStringBSTR([Runtime.InteropServices.Marshal]::SecureStringToBSTR($sec)) + if (-not $Pass) { $Pass = "pxe" } +} +$ApiBase = "http://${Server}:$WebPort" +$ExcludedDeployDirs = @("Operating Systems", "Out-of-box Drivers", "Packages") + +function Format-Size { + param([long]$Bytes) + if ($Bytes -ge 1GB) { return "{0:N1} GB" -f ($Bytes / 1GB) } + if ($Bytes -ge 1MB) { return "{0:N1} MB" -f ($Bytes / 1MB) } + return "{0:N0} KB" -f ($Bytes / 1KB) +} + +function Resolve-DestDir { + param([string]$Dir) + $d = $Dir -replace '^\*destinationdir\*\\?', '' + return (($d -replace '^[A-Za-z]:\\', '').Trim('\')) +} + +function Write-Step { + param([string]$Text) + Write-Host "" + Write-Host "========================================" -ForegroundColor Cyan + Write-Host $Text -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan +} + +function Stop-Cache { + # The cache itself is unusable: nothing was sent. + param([string[]]$Lines) + Write-Host "" + foreach ($l in $Lines) { Write-Host "ERROR: $l" -ForegroundColor Red } + Write-Host "Nothing was uploaded." -ForegroundColor Red + exit 2 +} + +function Read-JsonFile { + param([string]$Path) + return (Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json) +} + +# HTTP to the webapp with HttpWebRequest and NO proxy: the work PC's system +# proxy (Zscaler) must not see traffic to the isolated imaging LAN. +# Returns @{ Status = ; Body = }; never throws on HTTP errors. +function Invoke-Pxe { + param([string]$Method, [string]$Path, [byte[]]$Body, [string]$InFile, + [string]$ContentType = "application/json", [int]$TimeoutSec = 120) + $req = [System.Net.HttpWebRequest]::Create("$ApiBase$Path") + $req.Method = $Method + $req.Proxy = $null + $req.Timeout = $TimeoutSec * 1000 + $req.ReadWriteTimeout = $TimeoutSec * 1000 + $req.KeepAlive = $false + if ($Body -or $InFile) { + $req.ContentType = $ContentType + if ($InFile) { $req.ContentLength = (Get-Item -LiteralPath $InFile).Length } + else { $req.ContentLength = $Body.Length } + try { $rs = $req.GetRequestStream() } catch { return @{ Status = 0; Body = $_.Exception.Message } } + try { + if ($InFile) { + $fs = [System.IO.File]::OpenRead($InFile) + try { $fs.CopyTo($rs) } finally { $fs.Dispose() } + } else { + $rs.Write($Body, 0, $Body.Length) + } + } finally { $rs.Dispose() } + } + try { + $resp = $req.GetResponse() + } catch { + # A .NET method call wraps the WebException in a + # MethodInvocationException: look at the inner one. + $ex = $_.Exception + while ($ex -and -not ($ex -is [System.Net.WebException])) { $ex = $ex.InnerException } + if (-not $ex -or -not $ex.Response) { return @{ Status = 0; Body = $_.Exception.Message } } + $resp = $ex.Response + } + try { + $sr = New-Object System.IO.StreamReader($resp.GetResponseStream()) + $text = $sr.ReadToEnd() + $sr.Dispose() + return @{ Status = [int]$resp.StatusCode; Body = $text } + } finally { $resp.Close() } +} + +function Get-ErrorText { + param($Resp) + try { return ($Resp.Body | ConvertFrom-Json).error } catch { return $Resp.Body } +} + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " PXE Server Image Uploader" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +# --- Validate source paths --- +$CachePath = $CachePath.TrimEnd('\') +$DeployPath = Join-Path $CachePath "Deploy" +$ToolsPath = Join-Path (Split-Path $CachePath -Parent) "Tools" +if (-not (Test-Path $ToolsPath -PathType Container)) { + $ToolsPath = "C:\ProgramData\GEAerospace\MediaCreator\Tools" +} +$SourcesZip = Join-Path $CachePath "Boot\Sources.zip" +$ControlPath = Join-Path $DeployPath "Control" + +if (-not (Test-Path $DeployPath -PathType Container)) { + Write-Host "ERROR: Deploy directory not found at $DeployPath" -ForegroundColor Red + Write-Host " .\Upload-Image.ps1 -CachePath ""D:\Path\To\Cache""" -ForegroundColor Yellow + exit 2 +} + +# --- MCL finished? One tag at the cache root, written at the very end --- +$tags = @(Get-ChildItem -LiteralPath $CachePath -Filter "*.tag" -File -ErrorAction SilentlyContinue) +if ($tags.Count -eq 0) { + Stop-Cache @("No MCL completion tag (*.tag) in $CachePath.", + "MCL writes it only when a run finishes: the cache is incomplete or the run was interrupted.", + "Run Media Creator Lite again and let it finish.") +} +if ($tags.Count -gt 1) { + Stop-Cache (@("More than one MCL tag in ${CachePath}:") + ($tags | ForEach-Object { " $($_.Name)" }) + + @("MCL leaves exactly one. Run Media Creator Lite again.")) +} +$tag = $tags[0] +$tagBytes = [System.IO.File]::ReadAllBytes($tag.FullName) +if ($tagBytes.Length -lt 8) { Stop-Cache @("MCL tag $($tag.Name) is damaged ($($tagBytes.Length) bytes). Run MCL again.") } +# MCL writes DateTime.Now.Ticks (local time) into the tag (TagFactory). +try { $tagTime = New-Object DateTime ([BitConverter]::ToInt64($tagBytes, 0)) } +catch { Stop-Cache @("MCL tag $($tag.Name) holds no valid time. Run MCL again.") } +$tagOs = $tag.BaseName -replace '-(Production|QA)$', '' + +$late = @(Get-ChildItem -LiteralPath $ControlPath -File -ErrorAction SilentlyContinue | + Where-Object { $_.LastWriteTime -gt $tagTime.AddSeconds(5) }) +if ($late.Count -gt 0) { + Stop-Cache (@("These Control files are newer than the MCL finish tag ($tagTime):") + + ($late | ForEach-Object { " $($_.Name) $($_.LastWriteTime)" }) + + @("MCL was still writing or crashed. Run Media Creator Lite again and let it finish.")) +} + +# --- Every Control / Tools JSON must be non-empty and parse --- +$badJson = @() +foreach ($dir in @($ControlPath, $ToolsPath)) { + if (-not (Test-Path $dir -PathType Container)) { continue } + foreach ($j in Get-ChildItem -LiteralPath $dir -Filter "*.json" -File -Recurse) { + if ($j.Length -eq 0) { $badJson += "$($j.FullName) (0 bytes)"; continue } + try { $null = Read-JsonFile $j.FullName } catch { $badJson += "$($j.FullName) (not valid JSON)" } + } +} +if ($badJson.Count -gt 0) { + Stop-Cache (@("Broken JSON in the MCL cache:") + ($badJson | ForEach-Object { " $_" }) + + @("Delete the cache folder ($CachePath) and run Media Creator Lite again.")) +} + +# --- Parse user_selections.json --- +$SelectionsFile = Join-Path $ToolsPath "user_selections.json" +if (-not (Test-Path $SelectionsFile)) { + Write-Host "ERROR: user_selections.json not found at $SelectionsFile" -ForegroundColor Red + Write-Host " Run Media Creator Lite first to create a configuration." -ForegroundColor Yellow + exit 2 +} + +$selections = (Read-JsonFile $SelectionsFile)[0] +$selectedOsId = $selections.OperatingSystemSelection +$selectedModelIds = @($selections.HardwareModelSelection | ForEach-Object { $_.Id } | Select-Object -Unique) + +# --- Parse control JSONs --- +$osJsonFile = Join-Path $ControlPath "OperatingSystem.json" +$driverJsonFile = Join-Path $ControlPath "HardwareDriver.json" +$pkgJsonFile = Join-Path $ControlPath "packages.json" +$langJsonFile = Join-Path $ControlPath "LanguagePacks.json" + +if (-not (Test-Path $osJsonFile)) { + Write-Host "ERROR: OperatingSystem.json not found in $ControlPath" -ForegroundColor Red + exit 2 +} + +$osJson = @(Read-JsonFile $osJsonFile) +$driverJson = if (Test-Path $driverJsonFile) { @(Read-JsonFile $driverJsonFile) } else { @() } +$pkgJson = if (Test-Path $pkgJsonFile) { @(Read-JsonFile $pkgJsonFile) } else { @() } +$langJson = if (Test-Path $langJsonFile) { @(Read-JsonFile $langJsonFile) } else { @() } + +# --- The cached OS must be the selected one, and the one the tag names --- +$matchedOs = @($osJson | Where-Object { $_.operatingSystemVersion.id -eq [int]$selectedOsId }) +if ($matchedOs.Count -eq 0) { + Stop-Cache @("OperatingSystem.json does not hold the OS selected in user_selections.json (ID $selectedOsId).", + "Run Media Creator Lite again.") +} +$osv = $matchedOs[0].operatingSystemVersion +if (-not $tagOs.StartsWith($osv.productName, [StringComparison]::OrdinalIgnoreCase)) { + Stop-Cache @("The MCL tag says '$tagOs' but OperatingSystem.json holds '$($osv.productName)'.", + "The cache is a mix of two runs. Run Media Creator Lite again.") +} +$isLtsc = $osv.productName -match 'LTSC' +$suggested = if ($isLtsc) { "gea-shopfloor" } else { "standard-or-engineer" } + +# --- Resolve selections to paths --- +$osDirs = @() +$osTotalSize = [long]0 +foreach ($os in $matchedOs) { + $osDirs += Resolve-DestDir $os.operatingSystemVersion.wim.DestinationDir + $osTotalSize += $os.operatingSystemVersion.wim.size +} + +# Packages: enabled + matching OS ID (osId is a comma string in the new schema) +function Get-OsIds { param($e) if ($e.aOsIds) { return @($e.aOsIds) } return @("$($e.osId)" -split ',' | ForEach-Object { $_.Trim() }) } +$matchedPkgs = @($pkgJson | Where-Object { (Get-OsIds $_) -contains "$selectedOsId" -and $_.enabled -eq 1 }) +$pkgTotalSize = [long]0 +foreach ($pkg in $matchedPkgs) { $pkgTotalSize += $pkg.size } + +# Drivers: match selected model IDs (family) + OS ID, deduplicate by path +$allMatchingDrivers = @($driverJson | Where-Object { + $selectedModelIds -contains $_.family -and (Get-OsIds $_) -contains "$selectedOsId" +}) +$allDriverDirSet = [ordered]@{} +foreach ($drv in $allMatchingDrivers) { + $rel = Resolve-DestDir $drv.DestinationDir + if (-not $allDriverDirSet.Contains($rel)) { $allDriverDirSet[$rel] = $drv.size } +} +$allDriverCount = $allDriverDirSet.Count +$allDriverTotalSize = [long]0 +$allDriverDirSet.Values | ForEach-Object { $allDriverTotalSize += $_ } + +$driverDirs = @() +$driverTotalSize = [long]0 +if ($IncludeDrivers) { + $driverDirs = @($allDriverDirSet.Keys) + $driverTotalSize = $allDriverTotalSize +} + +# --- Display upload plan --- +Write-Host " Cache: $CachePath" +Write-Host " MCL run: $tagOs, finished $tagTime" -ForegroundColor Green +Write-Host " Server: $Server" +Write-Host " Target: $(if ($isLtsc) { 'Shop Floor (LTSC)' } else { 'Standard or Engineer (pick it in the webapp)' })" +Write-Host "" +Write-Host " Upload Plan (from user_selections.json):" -ForegroundColor Cyan +Write-Host " ------------------------------------------" +Write-Host " OS: $($osv.marketingName) $($osv.productName) ($(Format-Size $osTotalSize))" -ForegroundColor Green +Write-Host " Packages: $($matchedPkgs.Count) update(s) ($(Format-Size $pkgTotalSize))" -ForegroundColor Green +if ($IncludeDrivers) { + Write-Host " Drivers: $($driverDirs.Count) model(s) ($(Format-Size $driverTotalSize))" -ForegroundColor Green +} else { + Write-Host " Drivers: SKIPPED -- $allDriverCount available, use -IncludeDrivers" -ForegroundColor Yellow +} +Write-Host " Control: Always included" -ForegroundColor Gray +Write-Host " Tools: $(if (Test-Path $ToolsPath) { 'Yes' } else { 'Not found' })" -ForegroundColor $(if (Test-Path $ToolsPath) { "Gray" } else { "Yellow" }) +Write-Host " Sources: $(if (Test-Path $SourcesZip) { 'Yes (from Boot\Sources.zip)' } else { 'Not found' })" -ForegroundColor $(if (Test-Path $SourcesZip) { "Gray" } else { "Yellow" }) +Write-Host "" + +# --- Sources: extract now so it can be hashed with everything else --- +$TempExtract = $null +$TempSources = $null +if (Test-Path $SourcesZip) { + $TempExtract = Join-Path $env:TEMP "SourcesExtract" + Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue + Expand-Archive $SourcesZip -DestinationPath $TempExtract -Force + # Handle nested Sources folder (zip may contain Sources/ at root) + $TempSources = $TempExtract + if ((Test-Path (Join-Path $TempExtract "Sources")) -and -not (Test-Path (Join-Path $TempExtract "Diskpart"))) { + $TempSources = Join-Path $TempExtract "Sources" + } +} + +# --- Everything that will be sent: local file + path on the share --- +$sendList = New-Object System.Collections.Generic.List[object] +function Add-Tree { + param([string]$LocalDir, [string]$RelPrefix, [string[]]$SkipDirs = @()) + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { return } + $base = (Get-Item -LiteralPath $LocalDir).FullName.TrimEnd('\') + foreach ($f in Get-ChildItem -LiteralPath $base -Recurse -File -Force) { + $rel = $f.FullName.Substring($base.Length + 1) + if ($SkipDirs.Count -gt 0) { + # robocopy /XD skips that folder name at any depth + $segs = $rel.Split('\') + $skip = $false + for ($i = 0; $i -lt $segs.Count - 1; $i++) { if ($SkipDirs -contains $segs[$i]) { $skip = $true; break } } + if ($skip) { continue } + } + $sendList.Add([pscustomobject]@{ Local = $f.FullName; Rel = (Join-Path $RelPrefix $rel); Size = $f.Length }) + } +} +Add-Tree $DeployPath "Deploy" $ExcludedDeployDirs +foreach ($d in $osDirs) { Add-Tree (Join-Path $CachePath $d) $d } +foreach ($pkg in $matchedPkgs) { + $pdir = Resolve-DestDir $pkg.destinationDir + $rel = if ($pdir) { Join-Path $pdir $pkg.fileName } else { $pkg.fileName } + $local = Join-Path $CachePath $rel + if (Test-Path -LiteralPath $local -PathType Leaf) { + # File name as robocopy will write it (disk case), not as the JSON spells it + $item = Get-Item -LiteralPath $local + $rel = Join-Path (Split-Path $rel -Parent) $item.Name + $sendList.Add([pscustomobject]@{ Local = $item.FullName; Rel = $rel; Size = $item.Length }) + } +} +foreach ($d in $driverDirs) { Add-Tree (Join-Path $CachePath $d) $d } +Add-Tree $ToolsPath "Tools" +if ($TempSources) { Add-Tree $TempSources "Sources" } + +# The same file can be reached twice (an OS folder outside the excluded +# Deploy dirs, nested driver folders, two packages naming one file); a +# duplicate path would make the server refuse the whole manifest. +$seenRel = @{} +$dedup = New-Object System.Collections.Generic.List[object] +foreach ($f in $sendList) { + $k = ($f.Rel -replace '\\', '/').ToLower() + if (-not $seenRel.ContainsKey($k)) { $seenRel[$k] = $true; $dedup.Add($f) } +} +$sendList = $dedup + +# --- Hash everything; check payload against MCL's published hashes --- +Write-Step "Hashing $($sendList.Count) file(s) ($(Format-Size (($sendList | Measure-Object Size -Sum).Sum)))" +$mclHash = @{} +function Add-Mcl { param($Dest, $Name, $Hash) + # Only a real SHA-256 (as the server checks): anything else is not ours to judge + if (-not ($Dest -and $Name -and ("$Hash" -match '^[0-9a-fA-F]{64}$'))) { return } + $dd = Resolve-DestDir $Dest + $key = if ($dd) { Join-Path $dd $Name } else { "$Name" } + $mclHash[$key.ToLower()] = "$Hash".ToLower() } +foreach ($e in $osJson) { + $w = $e.operatingSystemVersion.wim + $n = if ($w.fileName) { $w.fileName } else { "install.wim" } + Add-Mcl $w.destinationDir $n $w.hash +} +foreach ($e in @($driverJson) + @($pkgJson)) { if ($e) { Add-Mcl $e.destinationDir $e.fileName $e.hash } } +foreach ($e in $langJson) { foreach ($lf in @($e.files)) { if ($lf) { Add-Mcl $lf.destFolder $lf.fileName $lf.hash } } } + +$totalBytes = [long](($sendList | Measure-Object Size -Sum).Sum) +$doneBytes = [long]0 +$badCache = @() +$i = 0 +foreach ($f in $sendList) { + $i++ + $pct = if ($totalBytes -gt 0) { [int](100 * $doneBytes / $totalBytes) } else { 100 } + Write-Progress -Activity "Hashing" -Status "$i/$($sendList.Count) $($f.Rel)" -PercentComplete $pct + $f | Add-Member -NotePropertyName Sha256 -NotePropertyValue (Get-FileHash -LiteralPath $f.Local -Algorithm SHA256).Hash.ToLower() + $doneBytes += $f.Size + $want = $mclHash[$f.Rel.ToLower()] + if ($want -and $want -ne $f.Sha256) { $badCache += $f } +} +Write-Progress -Activity "Hashing" -Completed +if ($badCache.Count -gt 0) { + Write-Host "" + Write-Host "These cache files do not match the hash MCL publishes for them (damaged download):" -ForegroundColor Red + foreach ($b in $badCache) { Write-Host " $($b.Rel)" -ForegroundColor Red } + Write-Host "" + Write-Host "MCL re-downloads a MISSING file correctly, but trusts a damaged one of the right size." -ForegroundColor Yellow + $ans = Read-Host "Delete these $($badCache.Count) file(s) from the MCL cache so MCL downloads them again? (Y/N)" + if ($ans -match '^[Yy]') { + foreach ($b in $badCache) { Remove-Item -LiteralPath $b.Local -Force; Write-Host " deleted $($b.Local)" -ForegroundColor Gray } + Write-Host "Now run Media Creator Lite again, then this script." -ForegroundColor Yellow + } + if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } + Write-Host "Nothing was uploaded." -ForegroundColor Red + exit 2 +} +Write-Host " All $($sendList.Count) file(s) hashed; payload matches MCL's published hashes." -ForegroundColor Green + +$pcName = ($env:COMPUTERNAME -replace '[^A-Za-z0-9._-]', '') +$uploadId = "{0}-{1}" -f (Get-Date -Format "yyyyMMdd-HHmmss"), $pcName +if ($uploadId.Length -gt 80) { $uploadId = $uploadId.Substring(0, 80) } +$manifest = [ordered]@{ + manifest_version = 1 + upload_id = $uploadId + created = (Get-Date -Format "yyyy-MM-ddTHH:mm:sszzz") + source_pc = $pcName + mcl = [ordered]@{ + tag = $tag.Name + finished = $tagTime.ToString("yyyy-MM-ddTHH:mm:ss") + os_id = [int]$osv.id + os_name = "$($osv.productName)" + os_version = "$($osv.versionNumber)" + build = "$($osv.buildNumber)" + suggested_target = $suggested + } + files = @($sendList | ForEach-Object { [ordered]@{ path = ($_.Rel -replace '\\', '/'); size = [long]$_.Size; sha256 = $_.Sha256 } }) +} +$manifestBytes = [System.Text.Encoding]::UTF8.GetBytes(($manifest | ConvertTo-Json -Depth 6 -Compress)) + +# --- Is the webapp reachable? (before spending time on the copy) --- +$ping = Invoke-Pxe -Method GET -Path "/api/upload/verify" -TimeoutSec 15 +if ($ping.Status -ne 200) { + Write-Host "ERROR: the PXE webapp at $ApiBase does not answer ($($ping.Status) $($ping.Body))." -ForegroundColor Red + Write-Host " The upload cannot be verified without it. Check the server, then run this again." -ForegroundColor Yellow + if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } + exit 1 +} + +# --- Connect to SMB share --- +Write-Host "" +Write-Host "Connecting to $Share ..." -ForegroundColor Gray +# net use goes through cmd: in Windows PowerShell 5.1 a native command's +# redirected stderr becomes an error record, which ErrorAction Stop turns +# into a crash ("The network connection could not be found"). +cmd /c "net use ""$Share"" /delete /y >nul 2>&1" | Out-Null + +$netResult = cmd /c "net use ""$Share"" /user:$User ""$Pass"" 2>&1" +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Could not connect to $Share" -ForegroundColor Red + Write-Host $netResult -ForegroundColor Red + Write-Host "" + Write-Host "Make sure:" -ForegroundColor Yellow + Write-Host " - The PXE server is running at $Server" -ForegroundColor Yellow + Write-Host " - This PC is on the 172.16.9.x imaging network" -ForegroundColor Yellow + Write-Host " - Samba is running on the PXE server" -ForegroundColor Yellow + if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } + exit 1 +} +Write-Host "Connected." -ForegroundColor Green + +$failed = $false +$stepNum = 0 +$totalSteps = 2 # Deploy base + OS always +if ($matchedPkgs.Count -gt 0) { $totalSteps++ } +if ($IncludeDrivers -and $driverDirs.Count -gt 0) { $totalSteps++ } +if (Test-Path $ToolsPath -PathType Container) { $totalSteps++ } +if ($TempSources) { $totalSteps++ } + +# robocopy exit codes 8 and up are failures +$rcArgs = @("/R:3", "/W:5", "/NP", "/ETA") + +# --- Step: Deploy base (Control, Applications, config -- skip big dirs) --- +$stepNum++ +Write-Step "[$stepNum/$totalSteps] Copying Deploy\ base (Control, Applications, config) ..." +robocopy $DeployPath "$Share\Deploy" /E /XD @ExcludedDeployDirs @rcArgs +if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Deploy base copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } + +# --- Step: Operating System --- +$stepNum++ +Write-Step "[$stepNum/$totalSteps] Copying Operating System ..." +foreach ($osDir in $osDirs) { + $src = Join-Path $CachePath $osDir + if (Test-Path -LiteralPath $src -PathType Container) { + Write-Host " $osDir" -ForegroundColor Gray + robocopy $src (Join-Path $Share $osDir) /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: OS copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } + } else { + Write-Host " SKIPPED (not cached): $osDir" -ForegroundColor Yellow + } +} + +# --- Step: Packages --- +if ($matchedPkgs.Count -gt 0) { + $stepNum++ + Write-Step "[$stepNum/$totalSteps] Copying Packages ..." + $pkgGroups = [ordered]@{} + foreach ($pkg in $matchedPkgs) { + $rel = Resolve-DestDir $pkg.destinationDir + if (-not $pkgGroups.Contains($rel)) { $pkgGroups[$rel] = @() } + $pkgGroups[$rel] += $pkg.fileName + } + foreach ($dir in $pkgGroups.Keys) { + $src = Join-Path $CachePath $dir + if (Test-Path -LiteralPath $src -PathType Container) { + foreach ($f in $pkgGroups[$dir]) { Write-Host " $f" -ForegroundColor Gray } + $robocopyArgs = @($src, (Join-Path $Share $dir)) + $pkgGroups[$dir] + $rcArgs + & robocopy @robocopyArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Package copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } + } else { + Write-Host " SKIPPED (not cached): $dir" -ForegroundColor Yellow + } + } +} + +# --- Step: Drivers (only with -IncludeDrivers) --- +if ($IncludeDrivers -and $driverDirs.Count -gt 0) { + $stepNum++ + Write-Step "[$stepNum/$totalSteps] Copying Drivers ($($driverDirs.Count) model(s)) ..." + $drvCopied = 0 + foreach ($drvDir in $driverDirs) { + $drvCopied++ + $src = Join-Path $CachePath $drvDir + if (Test-Path -LiteralPath $src -PathType Container) { + Write-Host " [$drvCopied/$($driverDirs.Count)] $drvDir" -ForegroundColor Gray + robocopy $src (Join-Path $Share $drvDir) /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Driver copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } + } else { + Write-Host " SKIPPED (not cached): $drvDir" -ForegroundColor Yellow + } + } +} + +# --- Step: Tools --- +if (Test-Path $ToolsPath -PathType Container) { + $stepNum++ + Write-Step "[$stepNum/$totalSteps] Copying Tools\ ..." + robocopy $ToolsPath "$Share\Tools" /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Tools copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } +} + +# --- Step: Sources --- +if ($TempSources) { + $stepNum++ + Write-Step "[$stepNum/$totalSteps] Copying Sources\ ..." + robocopy $TempSources "$Share\Sources" /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Sources copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } +} + +# --- Verify on the server; re-send what is wrong --- +$byRel = @{} +foreach ($f in $sendList) { $byRel[($f.Rel -replace '\\', '/').ToLower()] = $f } +$verified = $false +$result = $null +for ($round = 1; $round -le $MaxRounds + 1; $round++) { + Write-Step "Verifying on the server (round $round) ..." + $r = Invoke-Pxe -Method POST -Path "/api/upload/verify" -Body $manifestBytes -TimeoutSec 120 + if ($r.Status -ne 202) { + Write-Host "ERROR: verify request refused ($($r.Status)): $(Get-ErrorText $r)" -ForegroundColor Red + break + } + $deadline = (Get-Date).AddHours(2) + $postedAt = Get-Date + do { + if ((Get-Date) -gt $deadline) { $result = $null; break } + if ($result -and $result.upload_id -ne $uploadId -and ((Get-Date) - $postedAt).TotalSeconds -gt 120) { + Write-Host "ERROR: the server is working on another upload ($($result.upload_id)). Run this again later." -ForegroundColor Red + $result = $null; break + } + Start-Sleep -Seconds 2 + $s = Invoke-Pxe -Method GET -Path "/api/upload/verify" -TimeoutSec 30 + $result = $null + if ($s.Status -eq 200) { $result = $s.Body | ConvertFrom-Json } + if ($result -and $result.state -eq "running" -and $result.total_bytes -gt 0) { + Write-Progress -Activity "Server is hashing the upload" -Status "$($result.done_files)/$($result.total_files) files" ` + -PercentComplete ([Math]::Min(100, [int](100 * $result.done_bytes / $result.total_bytes))) + } + } while (-not $result -or $result.state -eq "running" -or $result.upload_id -ne $uploadId) + Write-Progress -Activity "Server is hashing the upload" -Completed + + if (-not $result) { Write-Host "ERROR: the server did not finish verifying within 2 hours." -ForegroundColor Red; break } + if ($result.state -eq "error") { Write-Host "ERROR: the server could not verify: $($result.error)" -ForegroundColor Red; break } + if ($result.ok) { $verified = $true; break } + + foreach ($b in @($result.bad_in_cache)) { if ($b) { Write-Host " BAD IN MCL CACHE: $($b.path) ($($b.reason))" -ForegroundColor Red } } + if (@($result.bad_in_cache).Count -gt 0) { break } # re-sending a bad source cannot help + + $redo = @(@($result.missing) + @($result.mismatch | ForEach-Object { $_.path }) | Where-Object { $_ }) + Write-Host " $($redo.Count) file(s) missing or different on the server." -ForegroundColor Yellow + if ($round -gt $MaxRounds) { break } + foreach ($rel in $redo) { + $f = $byRel[$rel.ToLower()] + if (-not $f) { continue } + if ($f.Size -le [long]$result.small_max) { + $q = "?upload_id=$([uri]::EscapeDataString($uploadId))&path=$([uri]::EscapeDataString($rel))" + $p = Invoke-Pxe -Method PUT -Path "/api/upload/file$q" -InFile $f.Local -ContentType "application/octet-stream" -TimeoutSec 300 + if ($p.Status -eq 200) { Write-Host " re-sent over HTTP: $rel" -ForegroundColor Green } + else { Write-Host " HTTP re-send failed ($($p.Status)) for ${rel}: $(Get-ErrorText $p)" -ForegroundColor Red } + } else { + $dstDir = Join-Path $Share (Split-Path ($f.Rel) -Parent) + $dstFile = Join-Path $dstDir (Split-Path $f.Local -Leaf) + Write-Host " re-copying (delete, then robocopy /J): $rel" -ForegroundColor Gray + # robocopy calls a same-size, same-time file "same" or "modified" + # and skips it, even with /IS /IT. Delete the bad copy so it is a + # new file; /IS /IT /IM stay as a backstop if the delete fails. + Remove-Item -LiteralPath $dstFile -Force -ErrorAction SilentlyContinue + $rc = robocopy (Split-Path $f.Local -Parent) $dstDir (Split-Path $f.Local -Leaf) /J /IS /IT /IM @rcArgs + if ($LASTEXITCODE -ge 8 -or $LASTEXITCODE -eq 0) { + # 0 = nothing copied: robocopy skipped it again + Write-Host " robocopy did not copy $rel (exit $LASTEXITCODE):" -ForegroundColor Red + $rc | Where-Object { $_ -match '\S' } | Select-Object -Last 8 | ForEach-Object { Write-Host " $_" } + } + } + } +} + +# --- Disconnect and clean up --- +cmd /c "net use ""$Share"" /delete /y >nul 2>&1" | Out-Null +if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } + +# --- Summary --- +Write-Host "" +if ($failed -and $verified) { + Write-Host " (robocopy reported errors on the first pass; the files it missed were re-sent and verified.)" -ForegroundColor Yellow +} +if ($verified) { + Write-Host "========================================" -ForegroundColor Green + Write-Host " Upload complete: all $($sendList.Count) file(s) verified on the server." -ForegroundColor Green + Write-Host "========================================" -ForegroundColor Green + if ([int]$result.extra_count -gt 0) { + Write-Host "" + Write-Host " NOTE: the upload folder also holds $($result.extra_count) file(s) from an earlier upload." -ForegroundColor Yellow + Write-Host " Import is blocked until you remove them (button on the Image Import page)." -ForegroundColor Yellow + } + Write-Host "" + Write-Host "Next steps:" -ForegroundColor Cyan + Write-Host " 1. Open the PXE webapp: http://${Server}:$WebPort/images/import" -ForegroundColor White + Write-Host " 2. Target: $(if ($isLtsc) { 'Shop Floor (the only choice for LTSC)' } else { 'Standard or Engineer' })" -ForegroundColor White + Write-Host " 3. Click Start import" -ForegroundColor White + Write-Host "" + exit 0 +} +Write-Host "========================================" -ForegroundColor Red +Write-Host " Upload NOT verified." -ForegroundColor Red +Write-Host "========================================" -ForegroundColor Red +if ($result) { + foreach ($m in @($result.missing)) { if ($m) { Write-Host " missing: $m" -ForegroundColor Red } } + foreach ($m in @($result.mismatch)) { if ($m) { Write-Host " differs: $($m.path) ($($m.reason))" -ForegroundColor Red } } +} +Write-Host "The webapp will not import this upload. Run the script again." -ForegroundColor Yellow +exit 1 diff --git a/scripts/diagnostics/Capture-LockdownState.ps1 b/scripts/diagnostics/Capture-LockdownState.ps1 index 9e76a09..91e9e34 100644 --- a/scripts/diagnostics/Capture-LockdownState.ps1 +++ b/scripts/diagnostics/Capture-LockdownState.ps1 @@ -474,6 +474,6 @@ Write-Host "" Write-Host "Snapshot complete: $out" Write-Host "" Write-Host "Next: copy that folder to the workstation. Easiest:" -Write-Host " net use Z: \\172.16.9.1\image-upload /user:pxe-upload pxe /persistent:no" +Write-Host " net use Z: \\172.16.9.1\image-upload /user:pxe-upload /persistent:no" Write-Host " robocopy `"$out`" `"Z:\state-$Stage-$stamp`" /E /NFL /NDL /NJH /NJS" Write-Host " net use Z: /delete /y" diff --git a/scripts/download-drivers.py b/scripts/download-drivers.py index 82c07ea..453b4d0 100755 --- a/scripts/download-drivers.py +++ b/scripts/download-drivers.py @@ -33,7 +33,7 @@ from pathlib import Path REPO_DIR = Path(__file__).resolve().parent PXE_HOST = "172.16.9.1" PXE_USER = "pxe" -PXE_PASS = "pxe" +PXE_PASS = os.environ.get("PXE_PASS") or "pxe" # the site's pxe_admin_password UPLOAD_DEST = "/home/pxe/image-upload" IMAGE_BASE = "/srv/samba/winpeapps" diff --git a/scripts/preflight.py b/scripts/preflight.py index fd53c78..9541487 100755 --- a/scripts/preflight.py +++ b/scripts/preflight.py @@ -37,7 +37,7 @@ REPO = os.path.dirname(HERE) PXE_HOST = "172.16.9.1" PXE_USER = "pxe" -PXE_PASS = "pxe" +PXE_PASS = os.environ.get("PXE_PASS") or "pxe" # the site's pxe_admin_password MODELS_TXT = "/srv/samba/winpeapps/_shared/BIOS/models.txt" IMAGE_BASE = "/srv/samba/winpeapps" diff --git a/scripts/share-drift.py b/scripts/share-drift.py index 6dfb4ea..e63e70f 100755 --- a/scripts/share-drift.py +++ b/scripts/share-drift.py @@ -40,7 +40,7 @@ import sys REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PXE_HOST = "172.16.9.1" PXE_USER = "pxe" -PXE_PASS = "pxe" +PXE_PASS = os.environ.get("PXE_PASS") or "pxe" # the site's pxe_admin_password GIT_OWNED, UNRECONCILED = "git-owned", "unreconciled" diff --git a/tests/vm/README.md b/tests/vm/README.md index 5f3809e..7d23751 100644 --- a/tests/vm/README.md +++ b/tests/vm/README.md @@ -70,7 +70,8 @@ Watch the server with `virt-viewer -c qemu:///system pxetest-server` (VNC on 127.0.0.1) or screenshots. On the installed system, first-boot output goes to `/var/log/first-boot.log`. Once the GE NIC takes a DHCP lease on the NAT network, `status.sh` shows its address and `ssh pxe@
` works from -the host. +the host (password `pxetest-admin`, or `PXETEST_ADMIN_PASSWORD` given to +build-image.sh). The offline package set must be built against the SAME ISO the image is built from (download-packages.sh resolves against that ISO's base). For a diff --git a/tests/vm/build-image.sh b/tests/vm/build-image.sh index ee4e311..123e6f2 100755 --- a/tests/vm/build-image.sh +++ b/tests/vm/build-image.sh @@ -11,6 +11,9 @@ # server with its GE NIC unplugged: virsh domif-setlink pxetest-server # 52:54:00:9e:10:01 down --config). # Env: PXETEST_PAYLOAD_ROOT (where offline-packages/, boot-files/... live) +# PXETEST_ADMIN_PASSWORD (the server's pxe login, default pxetest-admin) +# PXETEST_UPLOAD_PASSWORD / _BLANCCO_PASSWORD / _CLONEZILLA_PASSWORD +# (optional share password seeds) source "$(dirname "$0")/common.sh" SIZE=auto @@ -69,6 +72,16 @@ corp_nic_mac: "$SERVER_GE_MAC" # libvirt default NAT ("GE/corp") YML log "site vars: pxe_nic_mac=$SERVER_IMAGING_MAC corp_nic_mac=$SERVER_GE_MAC" fi +# The server's Linux login (required while it has the shipped "pxe") and, +# when set, seeds for the share accounts (Tools > Share accounts). +{ + echo "pxe_admin_password: \"${PXETEST_ADMIN_PASSWORD:-pxetest-admin}\"" + [ -n "${PXETEST_UPLOAD_PASSWORD:-}" ] && echo "pxe_upload_password: \"$PXETEST_UPLOAD_PASSWORD\"" + [ -n "${PXETEST_BLANCCO_PASSWORD:-}" ] && echo "blancco_smb_password: \"$PXETEST_BLANCCO_PASSWORD\"" + [ -n "${PXETEST_CLONEZILLA_PASSWORD:-}" ] && echo "clonezilla_smb_password: \"$PXETEST_CLONEZILLA_PASSWORD\"" + true +} >>"$SITE_VARS" +log "site vars: pxe_admin_password set (PXETEST_ADMIN_PASSWORD, default pxetest-admin)" "$REPO_ROOT/scripts/build-usb.sh" --yes --payload-root "$PXETEST_PAYLOAD_ROOT" \ --site-vars "$SITE_VARS" \ diff --git a/webapp/app.py b/webapp/app.py index 2a688aa..dbe7f35 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -58,6 +58,7 @@ from services import ( imaging_status, mcl_ppkg, nic_status, + share_accounts, system, unattend, upload_verify, @@ -72,6 +73,7 @@ from views import images as images_views from views import images_drivers as images_drivers_views from views import images_unattend as images_unattend_views from views import engineering_layer as engineering_layer_views +from views import share_accounts as share_accounts_views from views import ipxe as ipxe_views app = Flask(__name__) @@ -99,6 +101,7 @@ app.register_blueprint(images_views.bp) app.register_blueprint(images_drivers_views.bp) app.register_blueprint(images_unattend_views.bp) # Unattend review/save/undo app.register_blueprint(engineering_layer_views.bp) # Engineer image: Engineering Layer upload +app.register_blueprint(share_accounts_views.bp) # Tools > Share accounts (Samba passwords) # --------------------------------------------------------------------------- @@ -1605,6 +1608,13 @@ def _startnet_save_content(new_content, prior, action, detail): # still let the operator save (the raw editor is the source of truth). flash(f"Warning: could not snapshot prior startnet.cmd: {exc}", "warning") + # pxe-upload's password belongs to Tools > Share accounts: a restored + # snapshot or a hand edit cannot put an old one back. + pinned = share_accounts.pin_startnet(new_content) + if pinned != new_content: + flash("set SHAREPASS= was changed back to the pxe-upload password " + "(change it on Tools > Share accounts).", "warning") + new_content = pinned ok, err = wim.update_startnet(config.BOOT_WIM, new_content) if not ok: flash(f"Failed to update boot.wim: {err}", "danger") diff --git a/webapp/services/blancco_prefs.py b/webapp/services/blancco_prefs.py index f0224d7..6039d21 100644 --- a/webapp/services/blancco_prefs.py +++ b/webapp/services/blancco_prefs.py @@ -4,6 +4,8 @@ Three values differ per site and are filled into our preferences.xml: business_location printed on every erasure report (the site) wifi_ssid the Wi-Fi Blancco joins to license erasures in the GE wifi_password Blancco cloud (the imaging LAN has no internet) +A fourth, the password of the blancco Samba account in , +comes from services/share_accounts.py (Tools > Share accounts). They live in UPDATES_DIR/blancco/site-settings.json (0600, root). The repo's playbook/blancco-preferences.xml carries none of them; the playbook installs @@ -38,6 +40,10 @@ _LOCATION = re.compile(rb"()[^<]*()") _WIRELESS = re.compile( rb'(\s*[^<]*\s*)[^<]*' rb'(\s*)[^<]*()') +# The report share login: the blancco Samba account (Tools > Share accounts). +_SHARE = re.compile( + rb'(\s*blancco\s*' + rb')[^<]*()') def settings_path(): @@ -99,6 +105,8 @@ def apply(xml, settings=None): XML-escaped; the replacement is a function, so nothing in them is read as a regex group reference.""" s = settings or load() + from services import share_accounts + share_pw = escape(share_accounts.load()["blancco"]).encode("utf-8") loc = escape(s["business_location"] or "Unnamed site").encode("utf-8") ssid = escape(s["wifi_ssid"]).encode("utf-8") pw = escape(s["wifi_password"]).encode("utf-8") @@ -108,6 +116,9 @@ def apply(xml, settings=None): out, n = _WIRELESS.subn(lambda m: m.group(1) + ssid + m.group(2) + pw + m.group(3), out) if n != 1: raise ValueError("preferences have no single ssid/password block") + out, n = _SHARE.subn(lambda m: m.group(1) + share_pw + m.group(2), out) + if n != 1: + raise ValueError("preferences have no single login for blancco") return out diff --git a/webapp/services/blancco_update.py b/webapp/services/blancco_update.py index fca2a9d..69dd13a 100644 --- a/webapp/services/blancco_update.py +++ b/webapp/services/blancco_update.py @@ -988,6 +988,32 @@ def apply_prefs(version): return meta +def refresh_live_prefs(): + """Re-render the served preferences.xml (site settings, share login) + and write the same preferences into the live config.img, which the + BIOS (native) boot path reads instead. Returns True if a file changed. + release.json keeps describing the release when it did before.""" + from services import blancco_prefs + with _lock: + root = blancco_root() + meta_path = os.path.join(root, "release.json") + meta = uc.read_json(meta_path, None) + meta_ok = bool(meta and meta.get("fingerprint") == _fingerprint(root)) + changed = blancco_prefs.render() + cfg_path = os.path.join(root, "config.img") + cfg = _read(cfg_path) + if cfg is not None: + with open(blancco_prefs.served_prefs_path(), "rb") as f: + ours = f.read() + new = patch_config_img(cfg, ours) + if new != cfg: + _atomic_write(cfg_path, new) + changed = True + if changed and meta_ok: + _write_live_meta(meta) + return changed + + def save_merged_prefs(version, data): """A human-merged preferences.xml for a staged version: check it, keep it, and finish staging with it.""" diff --git a/webapp/services/share_accounts.py b/webapp/services/share_accounts.py new file mode 100644 index 0000000..77ef4bf --- /dev/null +++ b/webapp/services/share_accounts.py @@ -0,0 +1,255 @@ +"""Passwords of the three Samba accounts the imaging clients use. + + pxe-upload WinPE (startnet.cmd in boot.wim) maps the image and + enrollment shares with it; startnet writes it to + C:\\Enrollment\\fetch-source.txt for the Shop Floor first-logon + scripts (deleted by Invoke-PostPpkg.ps1 when imaging is + done). People use it to upload images and apps. + blancco Blancco saves erasure reports to [blancco-reports] (the + in our Blancco preferences). + clonezilla Clonezilla's back up / restore entries in the iPXE menu + (smb3://clonezilla:@server/clonezilla/). + +The stored values live in UPDATES_DIR/shares/accounts.json (0600, root) and +are the authority: apply() puts each one into Samba (smbpasswd) and into +every file that carries it, and the playbook runs apply at the end of every +run, so a re-run (which re-injects startnet.cmd from the USB and renders the +menu from site-vars) keeps them. The playbook seeds the file from site-vars +(pxe_upload_password, blancco_smb_password, clonezilla_smb_password) only +for accounts it does not hold yet. + +The WinPE image and the iPXE menu are served to any imaging client, so +anything on the imaging LAN can read pxe-upload's and clonezilla's +passwords. A site password still keeps other sites' PCs and people out. + + python3 -m services.share_accounts apply [--seed] + --seed reads SHARE_SEED_PXE_UPLOAD / _BLANCCO / _CLONEZILLA from the + environment (not argv: the passwords stay out of ps). +""" + +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timedelta + +import config +from services import updates_common as uc + +ACCOUNTS = ("pxe-upload", "blancco", "clonezilla") +DEFAULTS = {"pxe-upload": "pxe", "blancco": "blancco", "clonezilla": "clonezilla-backup"} +SEED_ENV = {"pxe-upload": "SHARE_SEED_PXE_UPLOAD", "blancco": "SHARE_SEED_BLANCCO", + "clonezilla": "SHARE_SEED_CLONEZILLA"} +# Each password sits in a cmd `set`, a `net use` line, XML, a smb3:// URI +# and a kernel command line: only characters none of them treat specially. +PASSWORD = re.compile(r"^[A-Za-z0-9._~-]{8,64}$") +PASSWORD_RULE = "8 to 64 characters from A-Z a-z 0-9 . _ ~ -" +# A PC that reported progress this recently is treated as still imaging. +BUSY_WINDOW = timedelta(hours=2) + +class ApplyFailed(Exception): + """The password is stored, but putting it somewhere failed.""" + + +_SHAREPASS = re.compile(r"^([ \t]*set[ \t]+SHAREPASS=)[^\r\n]*", re.IGNORECASE | re.MULTILINE) +_CZ_URI = re.compile(r"(smb3://clonezilla:)[^@\s\"]*(@)") + + +def state_path(): + return os.path.join(uc.state_dir("shares"), "accounts.json") + + +def _stored(): + data = uc.read_json(state_path(), None) + if not isinstance(data, dict): + return {} + return {a: data[a] for a in ACCOUNTS if isinstance(data.get(a), str) and data[a]} + + +def load(): + """Every account's current password (the default where none is stored).""" + out = dict(DEFAULTS) + out.update(_stored()) + return out + + +def _write(values): + path = state_path() + tmp = path + ".tmp" + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(values, f, indent=1) + os.chmod(tmp, 0o600) + os.replace(tmp, path) + + +def public(): + """For the page: never the passwords, only whether each is the default.""" + cur = load() + return [{"account": a, "default": cur[a] == DEFAULTS[a]} for a in ACCOUNTS] + + +def check(account, password): + if account not in ACCOUNTS: + raise ValueError(f"unknown account {account!r}") + if not isinstance(password, str) or not PASSWORD.match(password): + raise ValueError(f"the {account} password must be {PASSWORD_RULE}") + if password == DEFAULTS[account]: + raise ValueError(f"that is the shipped default for {account}, the same at every site") + + +def busy(): + """Sessions that look like a PC imaging now: running, updated within + BUSY_WINDOW. A password change mid-run breaks that PC's next net use.""" + from services import imaging_status + cutoff = datetime.now().astimezone() - BUSY_WINDOW + out = [] + for s in imaging_status.list_sessions(include_inferred=False): + if imaging_status.floor_state(s) != "running": + continue + try: + when = datetime.fromisoformat(s.get("last_updated") or "") + if when.tzinfo is None: + when = when.astimezone() + except (ValueError, TypeError): + continue + if when >= cutoff: + out.append(s.get("hostname") or s.get("serial") or s.get("mac") or "?") + return out + + +# --- where each password goes ---------------------------------------------- + +def _smbpasswd(account, password): + """Set the Samba password (the account itself is the playbook's).""" + r = subprocess.run(["smbpasswd", "-s", "-a", account], input=f"{password}\n{password}\n", + capture_output=True, text=True, timeout=30) + if r.returncode != 0: + raise RuntimeError(f"smbpasswd {account} failed: {(r.stderr or r.stdout).strip()}") + + +def pin_startnet(content, password=None): + """startnet.cmd text with `set SHAREPASS=` set to pxe-upload's password. + Every startnet save goes through this, so restoring an old snapshot + cannot bring an old password back.""" + pw = password if password is not None else load()["pxe-upload"] + return _SHAREPASS.sub(lambda m: m.group(1) + pw, content) + + +def _apply_startnet(password): + from services import wim + if not os.path.isfile(config.BOOT_WIM): + return "boot.wim: not on this server yet (skipped)" + cur = wim.extract_startnet(config.BOOT_WIM) + if cur is None: + raise RuntimeError("could not read startnet.cmd from boot.wim") + if not _SHAREPASS.search(cur): + raise RuntimeError("startnet.cmd in boot.wim has no 'set SHAREPASS=' line; " + "set the password in the Boot menu editor") + new = pin_startnet(cur, password) + if new == cur: + return "boot.wim: unchanged" + wim.save_snapshot(cur, note="SHARE_PASSWORD") + ok, err = wim.update_startnet(config.BOOT_WIM, new) + if not ok: + raise RuntimeError(f"boot.wim update failed: {err}") + return "boot.wim: startnet.cmd updated" + + +def _apply_menu(password): + from services import blancco_update as bu + path = bu.menu_path() + if not os.path.isfile(path): + return "iPXE menu: not on this server yet (skipped)" + with bu._lock: + path, text, _ = bu._read_menu() + if not _CZ_URI.search(text): + return "iPXE menu: no Clonezilla share entries (skipped)" + new = _CZ_URI.sub(lambda m: m.group(1) + password + m.group(2), text) + if new == text: + return "iPXE menu: unchanged" + backup = bu._write_menu(path, new, "pre-share-password") + # The backup sits in the web root: it must not keep the old password. + os.remove(backup) + return "iPXE menu: Clonezilla entries updated" + + +def _apply_blancco(): + from services import blancco_update as bu + return "Blancco preferences: " + ("updated" if bu.refresh_live_prefs() else "unchanged") + + +def apply(only=None, values=None): + """Put the stored passwords everywhere. Returns a list of what happened. + only: one account, or None for all three.""" + values = values or load() + done = [] + for account in ACCOUNTS: + if only and account != only: + continue + _smbpasswd(account, values[account]) + done.append(f"Samba {account}: set") + if account == "pxe-upload": + done.append(_apply_startnet(values[account])) + elif account == "blancco": + done.append(_apply_blancco()) + elif account == "clonezilla": + done.append(_apply_menu(values[account])) + return done + + +def set_password(account, password, force=False): + """Store one account's password and apply it. Refused while PCs image + unless force. Returns the list of what happened.""" + check(account, password) + if not force: + running = busy() + if running: + raise RuntimeError("PCs are imaging now (" + ", ".join(running[:5]) + + (", ..." if len(running) > 5 else "") + + "); wait for them to finish, or tick 'change anyway'") + stored = _stored() + stored[account] = password + _write(stored) + try: + return apply(only=account) + except Exception as e: + raise ApplyFailed(str(e)) from e + + +def seed_from_env(): + """Store site-vars values for accounts the file does not hold yet.""" + stored = _stored() + seeded = [] + for account in ACCOUNTS: + value = os.environ.get(SEED_ENV[account], "") + if account in stored or not value or value == DEFAULTS[account]: + continue + if not PASSWORD.match(value): + raise ValueError(f"site-vars password for {account} must be {PASSWORD_RULE}") + stored[account] = value + seeded.append(account) + if seeded: + _write(stored) + return seeded + + +def main(argv): + if not argv or argv[0] != "apply": + print(__doc__) + return 2 + if "--seed" in argv[1:]: + for a in seed_from_env(): + print(f"seeded {a} from site-vars") + for line in apply(): + print(line) + for row in public(): + if row["default"]: + print(f"WARNING: {row['account']} still has the shipped default password " + "(set one on Tools > Share accounts)") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/webapp/services/wim.py b/webapp/services/wim.py index 4dc66d9..27809ff 100644 --- a/webapp/services/wim.py +++ b/webapp/services/wim.py @@ -446,7 +446,7 @@ def _synth_block(item, ip, eol_indent=""): "echo Starting %s setup..." % label, r'start "FlatApp" %SYSTEMDRIVE%\GESetup\FlatSetupLoader.exe', r"for /l %%i in (1,1,2000000) do rem", - r"net use Z: \\%s\winpeapps\%s /user:pxe-upload pxe /persistent:no" + r"net use Z: \\%s\winpeapps\%s /user:%%SHAREUSER%% %%SHAREPASS%% /persistent:no" % (ip, image), "goto end", ] diff --git a/webapp/templates/base.html b/webapp/templates/base.html index b7c3591..4141088 100644 --- a/webapp/templates/base.html +++ b/webapp/templates/base.html @@ -78,6 +78,7 @@ {{ navlink('Boot menu', url_for('startnet_editor'), ep == 'startnet_editor') }} {{ navlink('Enrollment packages', url_for('enrollment'), ep == 'enrollment') }} {{ navlink('Clonezilla backups', url_for('clonezilla_backups'), ep == 'clonezilla_backups') }} + {{ navlink('Share accounts', url_for('share_accounts.page'), ep.startswith('share_accounts.')) }} {{ navlink('Audit log', url_for('audit_log'), ep == 'audit_log') }} diff --git a/webapp/templates/share_accounts.html b/webapp/templates/share_accounts.html new file mode 100644 index 0000000..fb23a4b --- /dev/null +++ b/webapp/templates/share_accounts.html @@ -0,0 +1,83 @@ +{% extends "base.html" %} +{% import '_ui.html' as ui %} +{% block title %}Share accounts - PXE Manager{% endblock %} + +{# Tools > Share accounts: one row per Samba account the imaging clients use, + each with its own password form. Passwords are never shown back. + services/share_accounts.py. #} + +{% block extra_head %} + +{% endblock %} + +{% block content %} +
+
+
Tools
+

Share accounts

+
The Samba logins the imaging PCs, Blancco and Clonezilla use on this server. Give each site its own: the shipped defaults are the same at every site.
+
+
+ +
+{% if busy %} +
+ {{ busy | length }} PC{{ 's' if busy | length != 1 }} imaging now ({{ busy[:5] | join(', ') }}{{ ', ...' if busy | length > 5 }}). A password change now breaks their next share connection, so it is refused unless you tick Change anyway. +
+{% endif %} + +
+

Accounts

+
Passwords: {{ rule }}. Saving changes Samba and every file that carries the password (boot.wim, the PXE boot menu, the Blancco preferences) at once. They are never shown here.
+ {% for a in accounts %} +
+
+ {{ a.account }} + {% if a.default %}{{ ui.chip('Default password', 'warn') }}{% else %}{{ ui.chip('Site password', 'ok') }}{% endif %} + {{ used_by[a.account] }} +
+
+ +
+ + +
+
+ + +
+ {% if busy %}{% endif %} + +
+
+ {% endfor %} +
+ +
+

Where else to change it

+
    +
  • pxe-upload: anyone who maps the shares by hand, and Upload-Image.ps1 (it asks for the password).
    • +
  • The server's own Linux login (pxe) is not set here: it is pxe_admin_password in site-vars.yml.
    • +
+
+
+{% endblock %} diff --git a/webapp/tests/test_share_accounts.py b/webapp/tests/test_share_accounts.py new file mode 100644 index 0000000..00c6007 --- /dev/null +++ b/webapp/tests/test_share_accounts.py @@ -0,0 +1,300 @@ +"""Tools > Share accounts: the pxe-upload, blancco and clonezilla Samba +passwords, and every file each one goes into. + +Run from the webapp folder: + PYTHONPATH=.: python -m unittest tests.test_share_accounts -v +""" +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest +from datetime import datetime, timedelta +from unittest import mock + +_T = tempfile.mkdtemp(prefix="pxe-shares-test-") +for _k, _sub in {"SAMBA_SHARE": "samba", "CLONEZILLA_SHARE": "cz", "BLANCCO_REPORTS": "br", + "ENROLLMENT_SHARE": "enr", "WEB_ROOT": "www", "IMAGING_DIR": "img", "UPDATES_DIR": "upd", + "TFTP_ROOT": "tftp", "BIOS_DIR": "bios", "UPLOAD_DIR": "upl", "SAMBA_LOG_DIR": "smblog", + "STARTNET_BACKUPS_DIR": "snb"}.items(): + os.environ.setdefault(_k, os.path.join(_T, _sub)) + os.makedirs(os.environ[_k], exist_ok=True) +for _k in ("AUDIT_LOG", "HARDWARE_STATE", "BLANCCO_STATE", "DNSMASQ_LEASES", "APACHE_ACCESS_LOG", + "DNSMASQ_SYSLOG", "NIC_STATUS_FILE", "NIC_CONF", "DNSMASQ_NIC_CONF", "DNSMASQ_CONF", "CLONEZILLA_STATE"): + os.environ.setdefault(_k, os.path.join(_T, _k.lower())) + +WEBAPP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO = os.path.dirname(WEBAPP) +sys.path.insert(0, WEBAPP) +os.chdir(WEBAPP) + +import config # noqa: E402 +import app as A # noqa: E402 +from services import blancco_prefs as BP # noqa: E402 +from services import blancco_update as BU # noqa: E402 +from services import share_accounts as SA # noqa: E402 +from services import wim # noqa: E402 + +PW = "Site-Pass_2026" + + +def newc(files): + """A newc cpio archive of {name: bytes} (what Blancco's config.img is).""" + out = b"" + def entry(name, data, mode): + nb = name.encode() + b"\0" + hdr = b"070701" + b"".join(b"%08X" % v for v in ( + 1, mode, 0, 0, 1, 0, len(data), 0, 0, 0, 0, len(nb), 0)) + blob = hdr + nb + blob += b"\0" * (-len(blob) % 4) + blob += data + b"\0" * (-len(data) % 4) + return blob + for name, data in files.items(): + out += entry(name, data, 0o100644) + return out + entry("TRAILER!!!", b"", 0) + + +def cpio_file(data, name): + return BU.cpio_member(data, name) + + +def startnet_text(): + with open(os.path.join(REPO, "playbook", "startnet.cmd"), encoding="utf-8") as f: + return f.read() + + +def menu_text(): + with open(os.path.join(REPO, "playbook", "ipxe", "GetPxeScript.aspx"), encoding="utf-8") as f: + return f.read().replace("__CLONEZILLA_SMB_PASSWORD__", "clonezilla-backup") + + +class Fixture(unittest.TestCase): + def setUp(self): + for d in (config.UPDATES_DIR, config.WEB_ROOT, config.IMAGING_DIR, wim.BACKUPS_DIR): + shutil.rmtree(d, ignore_errors=True) + os.makedirs(d) + # boot.wim with the repo startnet.cmd + src = tempfile.mkdtemp(dir=_T) + os.makedirs(os.path.join(src, "Windows", "System32")) + with open(os.path.join(src, "Windows", "System32", "startnet.cmd"), "w", newline="\r\n") as f: + f.write(startnet_text().replace("\r\n", "\n")) + os.makedirs(os.path.dirname(config.BOOT_WIM)) + subprocess.run(["wimcapture", src, config.BOOT_WIM, "--compress=none"], check=True, + capture_output=True) + # iPXE menu as the playbook renders it + os.makedirs(os.path.dirname(BU.menu_path())) + with open(BU.menu_path(), "w", newline="") as f: + f.write(menu_text()) + # Blancco: repo preferences, served copy, and a live config.img + os.makedirs(BU.blancco_root(), exist_ok=True) + with open(os.path.join(REPO, "playbook", "blancco-preferences.xml"), "rb") as f: + repo = f.read() + with open(BP.repo_prefs_path(), "wb") as f: + f.write(repo) + BP.render() + with open(BP.served_prefs_path(), "rb") as f: + served = f.read() + prefs_in_img = served + b"\0" * 3000 + b"\n" + self.cfg = newc({"report.key": b"KEY", "preferences.xml": prefs_in_img, "config.xml": b""}) + with open(os.path.join(BU.blancco_root(), "config.img"), "wb") as f: + f.write(self.cfg) + self.smb = mock.patch.object(SA, "_smbpasswd").start() + self.addCleanup(mock.patch.stopall) + + def startnet(self): + return wim.extract_startnet(config.BOOT_WIM) + + def menu(self): + with open(BU.menu_path()) as f: + return f.read() + + def prefs_share_pw(self, data): + m = re.search(rb'\s*blancco\s*' + rb'([^<]*)', data) + return m.group(1).decode() + + +class ShareAccounts(Fixture): + def test_defaults_and_public_hides_passwords(self): + self.assertEqual(SA.load(), SA.DEFAULTS) + pub = SA.public() + self.assertTrue(all(r["default"] for r in pub)) + self.assertNotIn("pxe", json.dumps(pub).replace("pxe-upload", "")) + + def test_rules(self): + for bad in ("short", "has space 123", "semi;colon12", "pct%PATH%xx", "a" * 65, "amp&ersand12"): + with self.assertRaises(ValueError, msg=bad): + SA.check("pxe-upload", bad) + with self.assertRaisesRegex(ValueError, "shipped default"): + SA.check("clonezilla", "clonezilla-backup") + with self.assertRaises(ValueError): + SA.check("root", PW) + SA.check("blancco", "Ok.pass_~-123") + + def test_pxe_upload_goes_into_samba_and_startnet(self): + done = SA.set_password("pxe-upload", PW) + self.smb.assert_called_once_with("pxe-upload", PW) + self.assertIn("boot.wim: startnet.cmd updated", done) + sn = self.startnet() + self.assertRegex(sn, r"(?m)^set SHAREPASS=%s\r$" % re.escape(PW)) + self.assertIn("set SHAREUSER=pxe-upload", sn) + self.assertEqual(sn.count("\r\n"), sn.count("\n")) # still CRLF + # Only that line changed. + a = startnet_text().replace("\r\n", "\n").splitlines() + b = sn.replace("\r\n", "\n").splitlines() + self.assertEqual([i for i, (x, y) in enumerate(zip(a, b)) if x != y], + [a.index("set SHAREPASS=pxe")]) + # The prior startnet was snapshotted. + self.assertTrue(any(s["note"] == "SHARE_PASSWORD" for s in wim.list_snapshots())) + # Stored privately. + st = os.stat(SA.state_path()) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) + self.assertEqual(SA.load()["pxe-upload"], PW) + self.assertFalse(SA.public()[0]["default"]) + + def test_clonezilla_goes_into_the_menu_without_a_backup_left(self): + before = set(os.listdir(os.path.dirname(BU.menu_path()))) + SA.set_password("clonezilla", PW) + m = self.menu() + self.assertEqual(m.count(f"smb3://clonezilla:{PW}@"), 2) + self.assertNotIn("clonezilla-backup", m) + self.assertEqual(set(os.listdir(os.path.dirname(BU.menu_path()))), before) + self.assertEqual(m, menu_text().replace("clonezilla:clonezilla-backup@", f"clonezilla:{PW}@")) + + def test_blancco_goes_into_served_prefs_and_live_config_img(self): + SA.set_password("blancco", PW) + with open(BP.served_prefs_path(), "rb") as f: + served = f.read() + self.assertEqual(self.prefs_share_pw(served), PW) + with open(os.path.join(BU.blancco_root(), "config.img"), "rb") as f: + cfg = f.read() + self.assertEqual(len(cfg), len(self.cfg)) + self.assertEqual(self.prefs_share_pw(cpio_file(cfg, "preferences.xml")), PW) + self.assertEqual(cpio_file(cfg, "report.key"), b"KEY") + # A release staged later gets it too (apply_prefs runs blancco_prefs.apply). + with open(BP.repo_prefs_path(), "rb") as f: + self.assertEqual(self.prefs_share_pw(BP.apply(f.read())), PW) + + def test_apply_is_idempotent(self): + SA.set_password("pxe-upload", PW) + SA.set_password("clonezilla", PW + "x") + SA.set_password("blancco", PW + "y") + done = SA.apply() + self.assertIn("boot.wim: unchanged", done) + self.assertIn("iPXE menu: unchanged", done) + self.assertIn("Blancco preferences: unchanged", done) + + def test_refused_while_a_pc_images(self): + now = datetime.now().astimezone() + with open(os.path.join(config.IMAGING_DIR, "ABC123.json"), "w") as f: + json.dump({"serial": "ABC123", "hostname": "BAY-7", "status": "in_progress", + "stage_index": 3, "last_updated": now.isoformat()}, f) + with open(os.path.join(config.IMAGING_DIR, "OLD1.json"), "w") as f: + json.dump({"serial": "OLD1", "status": "in_progress", "stage_index": 3, + "last_updated": (now - timedelta(hours=5)).isoformat()}, f) + self.assertEqual(SA.busy(), ["BAY-7"]) + with self.assertRaisesRegex(RuntimeError, "BAY-7"): + SA.set_password("pxe-upload", PW) + self.assertEqual(SA.load()["pxe-upload"], "pxe") # nothing stored + self.smb.assert_not_called() + SA.set_password("pxe-upload", PW, force=True) + self.assertEqual(SA.load()["pxe-upload"], PW) + + def test_seed_only_for_accounts_not_held(self): + SA.set_password("blancco", PW) + env = {"SHARE_SEED_PXE_UPLOAD": "Seed-Upload-1", "SHARE_SEED_BLANCCO": "Seed-Blancco-1", + "SHARE_SEED_CLONEZILLA": "clonezilla-backup"} + with mock.patch.dict(os.environ, env): + self.assertEqual(SA.seed_from_env(), ["pxe-upload"]) + cur = SA.load() + self.assertEqual((cur["pxe-upload"], cur["blancco"], cur["clonezilla"]), + ("Seed-Upload-1", PW, "clonezilla-backup")) + with mock.patch.dict(os.environ, {"SHARE_SEED_CLONEZILLA": "bad pw"}): + with self.assertRaises(ValueError): + SA.seed_from_env() + + def test_cli_apply_seed(self): + with mock.patch.dict(os.environ, {"SHARE_SEED_CLONEZILLA": "Seed-Clone-1"}): + self.assertEqual(SA.main(["apply", "--seed"]), 0) + self.assertIn("smb3://clonezilla:Seed-Clone-1@", self.menu()) + self.assertEqual(self.smb.call_count, 3) + + def test_failure_after_store_is_reported_as_such(self): + self.smb.side_effect = RuntimeError("smbpasswd pxe-upload failed: no such user") + with self.assertRaises(SA.ApplyFailed): + SA.set_password("pxe-upload", PW) + + def test_no_server_files_yet(self): + os.remove(config.BOOT_WIM) + os.remove(BU.menu_path()) + os.remove(os.path.join(BU.blancco_root(), "config.img")) + done = SA.apply() + self.assertTrue(any("skipped" in d for d in done)) + + +class StartnetPinning(Fixture): + def setUp(self): + super().setUp() + self.c = A.app.test_client() + html = self.c.get("/startnet").get_data(as_text=True) + self.tok = re.search(r'name="csrf-token" content="([^"]+)"', html).group(1) + + def test_editor_save_cannot_bring_back_an_old_password(self): + SA.set_password("pxe-upload", PW) + old = startnet_text() # carries SHAREPASS=pxe, like an old snapshot + r = self.c.post("/startnet/save", data={"content": old, "_csrf_token": self.tok}) + self.assertIn(r.status_code, (200, 302)) + self.assertRegex(self.startnet(), r"(?m)^set SHAREPASS=%s\r$" % re.escape(PW)) + + def test_new_menu_block_uses_the_variables(self): + blk = wim._synth_block({"target": "x", "target_image": "gea-x", "label": "X"}, "172.16.9.1") + self.assertIn(r"net use Z: \\172.16.9.1\winpeapps\gea-x /user:%SHAREUSER% %SHAREPASS% /persistent:no", blk) + + +class Page(Fixture): + def setUp(self): + super().setUp() + self.c = A.app.test_client() + + def tok(self): + html = self.c.get("/tools/share-accounts/").get_data(as_text=True) + return re.search(r'name="csrf-token" content="([^"]+)"', html).group(1) + + def test_page_and_save(self): + r = self.c.get("/tools/share-accounts/") + self.assertEqual(r.status_code, 200) + html = r.get_data(as_text=True) + for a in SA.ACCOUNTS: + self.assertIn(a, html) + self.assertIn("Default password", html) + tok = self.tok() + r = self.c.post("/tools/share-accounts/clonezilla", + data={"password": PW, "confirm": PW, "_csrf_token": tok}, follow_redirects=True) + html = r.get_data(as_text=True) + self.assertIn("clonezilla password changed", html) + self.assertNotIn(PW, html) + self.assertIn(f"clonezilla:{PW}@", self.menu()) + + def test_mismatch_and_bad_password(self): + tok = self.tok() + r = self.c.post("/tools/share-accounts/blancco", + data={"password": PW, "confirm": PW + "z", "_csrf_token": tok}, follow_redirects=True) + self.assertIn("differ", r.get_data(as_text=True)) + r = self.c.post("/tools/share-accounts/blancco", + data={"password": "bad", "confirm": "bad", "_csrf_token": tok}, follow_redirects=True) + self.assertIn("not changed", r.get_data(as_text=True)) + self.assertEqual(SA.load()["blancco"], "blancco") + self.smb.assert_not_called() + + def test_csrf_required(self): + r = self.c.post("/tools/share-accounts/blancco", data={"password": PW, "confirm": PW}) + self.assertIn(r.status_code, (400, 403)) + self.assertEqual(SA.load()["blancco"], "blancco") + + +if __name__ == "__main__": + unittest.main() diff --git a/webapp/views/blancco_update.py b/webapp/views/blancco_update.py index d8eec53..668e934 100644 --- a/webapp/views/blancco_update.py +++ b/webapp/views/blancco_update.py @@ -45,14 +45,16 @@ def page(): @bp.route("/site-settings", methods=["POST"]) def site_settings(): """Business location and licensing Wi-Fi: saved, then the served - preferences.xml is rendered again. The password is never shown back.""" + preferences.xml and the live config.img are written again. The password + is never shown back.""" try: with bu._lock: blancco_prefs.save(request.form.get("business_location", ""), request.form.get("wifi_ssid", ""), request.form.get("wifi_password", "")) - changed = blancco_prefs.render() - except (OSError, ValueError) as e: + # Served preferences.xml and the live config.img (BIOS path). + changed = bu.refresh_live_prefs() + except (OSError, ValueError, bu.StageBlocked) as e: flash(f"Blancco site settings not saved: {e}", "danger") return _back() s = blancco_prefs.public() diff --git a/webapp/views/share_accounts.py b/webapp/views/share_accounts.py new file mode 100644 index 0000000..5ce311d --- /dev/null +++ b/webapp/views/share_accounts.py @@ -0,0 +1,43 @@ +"""Tools > Share accounts: the Samba passwords the imaging clients use +(pxe-upload, blancco, clonezilla). Logic in services/share_accounts.py.""" + +from flask import Blueprint, flash, redirect, render_template, request, url_for + +from services import share_accounts as sa +from services.audit import audit + +bp = Blueprint("share_accounts", __name__, url_prefix="/tools/share-accounts") + +USED_BY = { + "pxe-upload": "WinPE and Shop Floor setup on every imaging PC; image and app uploads", + "blancco": "Blancco saving erasure reports to the blancco-reports share", + "clonezilla": "Clonezilla back up and restore (in the PXE boot menu)", +} + + +@bp.route("/") +def page(): + return render_template("share_accounts.html", accounts=sa.public(), used_by=USED_BY, + busy=sa.busy(), rule=sa.PASSWORD_RULE) + + +@bp.route("/", methods=["POST"]) +def save(account): + pw = request.form.get("password", "") + if pw != request.form.get("confirm", ""): + flash(f"{account}: the two passwords differ; nothing changed.", "danger") + return redirect(url_for("share_accounts.page")) + force = request.form.get("force") == "1" + try: + done = sa.set_password(account, pw, force=force) + except sa.ApplyFailed as e: + audit("SHARE_PASSWORD_FAILED", f"{account}: {e}") + flash(f"{account} password stored, but applying it failed: {e}. " + "Fix the cause and save it again.", "danger") + return redirect(url_for("share_accounts.page")) + except (ValueError, RuntimeError, OSError) as e: + flash(f"{account} password not changed: {e}", "danger") + return redirect(url_for("share_accounts.page")) + audit("SHARE_PASSWORD", f"{account}{' (while PCs were imaging)' if force else ''}: " + "; ".join(done)) + flash(f"{account} password changed. " + "; ".join(done) + ".", "success") + return redirect(url_for("share_accounts.page"))