diff --git a/scripts/windows/Sync-BPRT.ps1 b/scripts/windows/Sync-BPRT.ps1 new file mode 100644 index 0000000..0c5c3eb --- /dev/null +++ b/scripts/windows/Sync-BPRT.ps1 @@ -0,0 +1,445 @@ +<# +.SYNOPSIS + Upload the new and changed files of the vendor BPRT folder to the PXE server. + +.DESCRIPTION + Run on the GE work PC where MediaCreator Lite keeps the vendor's BPRT + folder (the MCL provisioning packages, the Office zips and the bulk-enroll + scripts). The PXE server is reachable from there only over the GE network: + its webapp and SSH, no SMB. + + 1. GET /updates/bprt/manifest.json: path, size and sha256 of every + vendor file in the server's live BPRT. The first request after a big + change makes the server hash 7.7 GB; it answers 503 until then and + this script waits. + 2. Hash the local folder (Get-FileHash SHA256) and parse-check every + PowerShell file (a corrupt MediaCreator cache once held scripts with + stale tails), and check every .ppkg has a WIM header. + 3. Write sync.json (the complete intended vendor file list with sizes and + hashes) and upload the new and changed files plus sync.json with ONE + sftp session into BPRT-incoming/.part on the server, renamed to + when complete. You type the password once. + 4. Print the webapp page where the batch is validated and activated. + + Nothing on the server changes until the batch is activated on that page. + The site's own files in BPRT (our Select-PCConfig.ps1 shim, the + Register-BulkEnrollTask*.ps1 scripts) are not in the manifest and are + never uploaded; the vendor's Select-PCConfig.ps1 is compared with the copy + the server keeps as Select-PCConfig-vendor.ps1. + + Windows PowerShell 5.1, built-in modules and the built-in OpenSSH client + (sftp.exe) only. + +.PARAMETER Source + The vendor BPRT folder. Default: the MediaCreator Lite cache. + +.PARAMETER WebUrl + The PXE webapp on the GE network. + +.PARAMETER SshHost + The PXE server's GE-network address for sftp. + +.PARAMETER SshUser + SSH login (password prompt). + +.PARAMETER DryRun + Compare and write sync.json and the sftp batch file, print what would be + uploaded, and stop before connecting with sftp. The manifest is still + read from the webapp when it answers (read-only); use -ManifestFile to + work fully offline. + +.PARAMETER ZipOut + Instead of sftp, write the changed files plus sync.json into this zip, to + upload on the webapp page (the fallback when SSH is not available). + +.PARAMETER ManifestFile + Use a saved manifest.json instead of fetching it. + +.PARAMETER Force + Upload even when a script fails to parse or a package has no WIM header. + The webapp still refuses to activate such a batch. + +.PARAMETER MaxWaitMinutes + How long to wait for the server to finish hashing its BPRT. + +.PARAMETER UseSystemProxy + Reach the webapp through the system proxy (default: direct, as for any + 10.x address). + +.EXAMPLE + powershell -ExecutionPolicy Bypass -File .\Sync-BPRT.ps1 -DryRun + +.EXAMPLE + powershell -ExecutionPolicy Bypass -File .\Sync-BPRT.ps1 + +.EXAMPLE + powershell -ExecutionPolicy Bypass -File .\Sync-BPRT.ps1 -ZipOut C:\Temp\bprt.zip +#> + +[CmdletBinding()] +param( + [string]$Source = 'C:\ProgramData\GEAerospace\MediaCenter\Cache\Deploy\Applications\BPRT', + [string]$WebUrl = 'http://10.134.48.170:9009', + [string]$SshHost = '10.134.48.170', + [string]$SshUser = 'pxe', + [switch]$DryRun, + [string]$ZipOut, + [string]$ManifestFile, + [switch]$Force, + [int]$MaxWaitMinutes = 30, + [switch]$UseSystemProxy +) + +$ErrorActionPreference = 'Stop' + +# The cache folder has also been seen under MediaCreator rather than +# MediaCenter; try that one when the default is missing. +$AltSource = 'C:\ProgramData\GEAerospace\MediaCreator\Cache\Deploy\Applications\BPRT' +# Files never sent: Windows litter. Names with [ or ] would be read as +# wildcards by sftp's put. +$SkipNames = @('Thumbs.db', 'desktop.ini', '.DS_Store') +$ScriptExts = @('.ps1', '.psm1', '.psd1') +$WimMagic = [byte[]](0x4D, 0x53, 0x57, 0x49, 0x4D, 0x00, 0x00, 0x00) # "MSWIM\0\0\0" + +function Write-Step([string]$Text) { Write-Host ''; Write-Host "== $Text" -ForegroundColor Cyan } + +function Format-Size([double]$Bytes) { + if ($Bytes -ge 1GB) { return ('{0:N2} GB' -f ($Bytes / 1GB)) } + if ($Bytes -ge 1MB) { return ('{0:N0} MB' -f ($Bytes / 1MB)) } + if ($Bytes -ge 1KB) { return ('{0:N0} KB' -f ($Bytes / 1KB)) } + return ('{0} B' -f $Bytes) +} + +function Get-Prefix([string]$Name) { + # Tenant_Environment_Purpose, the part the enroll scripts match on. + return (($Name -split '_')[0..2] -join '_') +} + +function Get-PpkgVersion([string]$Name) { + if ($Name -match '_(v\d+\.\d+)\.ppkg$') { return $Matches[1] } + return '?' +} + +function Find-WebException($Err) { + $ex = $Err.Exception + while ($ex -and -not ($ex -is [System.Net.WebException])) { $ex = $ex.InnerException } + return $ex +} + +function Get-Manifest { + param([string]$Url) + $deadline = (Get-Date).AddMinutes($MaxWaitMinutes) + while ($true) { + $req = [System.Net.HttpWebRequest]::Create($Url) + $req.Timeout = 60000 + $req.ReadWriteTimeout = 60000 + if (-not $UseSystemProxy) { $req.Proxy = $null } + $resp = $null + try { + $resp = $req.GetResponse() + } catch { + $wex = Find-WebException $_ + if ($wex -and $wex.Response) { $resp = $wex.Response } + else { throw "Cannot reach $Url : $($_.Exception.Message)" } + } + $status = [int]$resp.StatusCode + $retry = 30 + $ra = $resp.Headers['Retry-After'] + if ($ra -and ($ra -as [int])) { $retry = [int]$ra } + try { + $reader = New-Object System.IO.StreamReader($resp.GetResponseStream(), [System.Text.Encoding]::UTF8) + $body = $reader.ReadToEnd() + $reader.Close() + } finally { + $resp.Close() + } + if ($status -eq 200) { return ($body | ConvertFrom-Json) } + if ($status -eq 503) { + $msg = 'The server is busy.' + try { + $j = $body | ConvertFrom-Json + if ($j.message) { $msg = $j.message } + if ($null -ne $j.percent) { $msg = "$msg ($($j.percent)%)" } + } catch { } + if ((Get-Date) -gt $deadline) { + throw "Gave up after $MaxWaitMinutes minutes: $msg Run this again later." + } + Write-Host (" {0} Asking again in {1} s." -f $msg, $retry) + Start-Sleep -Seconds $retry + continue + } + throw "HTTP $status from $Url : $body" + } +} + +function ConvertTo-SftpPath([string]$Path) { + # sftp batch arguments: forward slashes (the Windows client takes them), + # in double quotes so spaces survive ("M365 Office x86.zip"). + return '"' + ($Path -replace '\\', '/') + '"' +} + +function Write-Utf8NoBom([string]$Path, [string]$Text) { + [System.IO.File]::WriteAllText($Path, $Text, (New-Object System.Text.UTF8Encoding($false))) +} + +try { + # --- source folder ---------------------------------------------------------- + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if (-not $PSBoundParameters.ContainsKey('Source') -and (Test-Path -LiteralPath $AltSource -PathType Container)) { + $Source = $AltSource + } else { + throw "Source folder not found: $Source (use -Source)" + } + } + $Source = (Resolve-Path -LiteralPath $Source).ProviderPath.TrimEnd('\', '/') + $WebUrl = $WebUrl.TrimEnd('/') + Write-Host "Source: $Source" + Write-Host "Server: $WebUrl (sftp $SshUser@$SshHost)" + + # --- server manifest -------------------------------------------------------- + Write-Step 'Reading the server manifest' + $manifest = $null + if ($ManifestFile) { + $manifest = Get-Content -LiteralPath $ManifestFile -Raw -Encoding UTF8 | ConvertFrom-Json + Write-Host " from $ManifestFile" + } else { + try { + $manifest = Get-Manifest -Url "$WebUrl/updates/bprt/manifest.json" + } catch { + if (-not $DryRun) { throw } + Write-Warning "$($_.Exception.Message)" + Write-Warning 'Dry run without a manifest: every file counts as new.' + } + } + $server = @{} # vendor path -> manifest row (PowerShell hashtables ignore case, like Windows) + $neverSend = @{} # site files on the server, never uploaded + $incoming = '/srv/samba/winpeapps/_shared/Applications/BPRT-incoming' + $finishUrl = "$WebUrl/updates/bprt/" + if ($manifest) { + foreach ($f in @($manifest.files)) { if ($f) { $server[$f.path] = $f } } + foreach ($n in @($manifest.overlay)) { if ($n) { $neverSend[$n] = $true } } + if ($manifest.incoming) { $incoming = $manifest.incoming } + if ($manifest.finish_url) { $finishUrl = $manifest.finish_url } + Write-Host (" {0} vendor files on the server, {1}, manifest of {2}" -f $server.Count, (Format-Size $manifest.bytes), $manifest.generated) + } + + # --- local files ------------------------------------------------------------ + Write-Step 'Hashing the local folder' + $all = @(Get-ChildItem -LiteralPath $Source -Recurse -File -Force | Sort-Object FullName) + $local = New-Object System.Collections.Generic.List[object] + $skipped = New-Object System.Collections.Generic.List[string] + $badNames = New-Object System.Collections.Generic.List[string] + $totalBytes = [double]0 + foreach ($fi in $all) { + $rel = $fi.FullName.Substring($Source.Length).TrimStart('\', '/') -replace '\\', '/' + if ($SkipNames -contains $fi.Name) { $skipped.Add($rel); continue } + if ($neverSend.ContainsKey($rel)) { $skipped.Add("$rel (site file on the server)"); continue } + if ($rel -match '[\[\]]') { $badNames.Add($rel) } + $local.Add([pscustomobject]@{ Path = $rel; Full = $fi.FullName; Size = [int64]$fi.Length; Sha = '' }) + $totalBytes += $fi.Length + } + if ($local.Count -eq 0) { throw "No files in $Source" } + $done = [double]0 + $sw = [System.Diagnostics.Stopwatch]::StartNew() + foreach ($f in $local) { + $pct = 0 + if ($totalBytes -gt 0) { $pct = [int](100 * $done / $totalBytes) } + Write-Progress -Activity 'Hashing BPRT' -Status ("{0} ({1})" -f $f.Path, (Format-Size $f.Size)) -PercentComplete $pct + $f.Sha = (Get-FileHash -LiteralPath $f.Full -Algorithm SHA256).Hash.ToUpperInvariant() + $done += $f.Size + } + Write-Progress -Activity 'Hashing BPRT' -Completed + Write-Host (" {0} files, {1}, hashed in {2:N0} s" -f $local.Count, (Format-Size $totalBytes), $sw.Elapsed.TotalSeconds) + foreach ($s in $skipped) { Write-Host " skipped $s" } + + # --- sanity checks ---------------------------------------------------------- + Write-Step 'Checking scripts and packages' + $problems = New-Object System.Collections.Generic.List[object] + foreach ($f in $local) { + $ext = [System.IO.Path]::GetExtension($f.Path).ToLowerInvariant() + if ($ScriptExts -contains $ext) { + $tokens = $null + $errs = $null + [void][System.Management.Automation.Language.Parser]::ParseFile($f.Full, [ref]$tokens, [ref]$errs) + if ($errs -and $errs.Count -gt 0) { + $first = $errs[0] + $problems.Add([ordered]@{ path = $f.Path; error = ("line {0}: {1} ({2} error(s))" -f $first.Extent.StartLineNumber, $first.Message, $errs.Count) }) + } + } elseif ($ext -eq '.ppkg') { + $head = New-Object byte[] 8 + $fs = [System.IO.File]::OpenRead($f.Full) + try { $n = $fs.Read($head, 0, 8) } finally { $fs.Close() } + $ok = ($n -eq 8) + for ($i = 0; $ok -and $i -lt 8; $i++) { if ($head[$i] -ne $WimMagic[$i]) { $ok = $false } } + if (-not $ok) { $problems.Add([ordered]@{ path = $f.Path; error = 'no WIM header: not a provisioning package' }) } + } + } + foreach ($p in $problems) { Write-Host (" PROBLEM {0}: {1}" -f $p.path, $p.error) -ForegroundColor Red } + foreach ($b in $badNames) { Write-Host " PROBLEM $b : sftp would read [ ] in the name as a wildcard" -ForegroundColor Red } + if ($problems.Count -eq 0 -and $badNames.Count -eq 0) { Write-Host ' all scripts parse, all packages have a WIM header' } + if ($badNames.Count -gt 0) { throw 'Rename the files listed above, or upload a zip on the webapp page.' } + if ($problems.Count -gt 0 -and -not $Force) { + throw ('{0} file(s) look corrupt. If the MediaCreator cache is damaged, delete it and let MediaCreator Lite download again. -Force uploads anyway, but the server refuses such a batch; it is only for looking at the files there.' -f $problems.Count) + } + + # --- compare ---------------------------------------------------------------- + $changed = New-Object System.Collections.Generic.List[object] + $localPaths = @{} + foreach ($f in $local) { + $localPaths[$f.Path] = $true + $s = $server[$f.Path] + $same = $false + if ($s) { + $same = ($f.Sha -eq ([string]$s.sha256).ToUpperInvariant()) + if (-not $same -and $s.sha256_nobom) { $same = ($f.Sha -eq ([string]$s.sha256_nobom).ToUpperInvariant()) } + } + if (-not $same) { $changed.Add($f) } + } + $removed = @($server.Keys | Where-Object { -not $localPaths.ContainsKey($_) } | Sort-Object) + $uploadBytes = [double]0 + foreach ($f in $changed) { $uploadBytes += $f.Size } + + Write-Step 'Differences' + foreach ($f in $changed) { + $tag = 'changed' + if (-not $server.ContainsKey($f.Path)) { $tag = 'new' } + Write-Host (" {0,-8} {1} ({2})" -f $tag, $f.Path, (Format-Size $f.Size)) + } + foreach ($r in $removed) { Write-Host (" {0,-8} {1} (not in the local folder; activation removes it from BPRT)" -f 'removed', $r) -ForegroundColor Yellow } + # Package versions per tenant/environment/purpose. + $oldPk = @{} + $newPk = @{} + foreach ($k in $server.Keys) { if ($k -like '*.ppkg') { $oldPk[(Get-Prefix $k)] = @($oldPk[(Get-Prefix $k)]) + $k } } + foreach ($f in $local) { if ($f.Path -like '*.ppkg') { $newPk[(Get-Prefix $f.Path)] = @($newPk[(Get-Prefix $f.Path)]) + $f.Path } } + $prefixes = @(@($oldPk.Keys) + @($newPk.Keys) | Sort-Object -Unique) + foreach ($p in $prefixes) { + $o = (@($oldPk[$p] | Where-Object { $_ }) | ForEach-Object { Get-PpkgVersion $_ }) -join ', ' + $n = (@($newPk[$p] | Where-Object { $_ }) | ForEach-Object { Get-PpkgVersion $_ }) -join ', ' + if ($o -ne $n) { + if (-not $o) { $o = 'none' } + if (-not $n) { $n = 'none' } + Write-Host (" package {0}: {1} to {2}" -f $p, $o, $n) -ForegroundColor Green + } + } + Write-Host (" {0} new or changed ({1}), {2} removed, {3} unchanged" -f $changed.Count, (Format-Size $uploadBytes), $removed.Count, ($local.Count - $changed.Count)) + + if ($manifest -and $changed.Count -eq 0 -and $removed.Count -eq 0) { + Write-Host '' + Write-Host 'The server already has exactly this BPRT. Nothing to upload.' -ForegroundColor Green + exit 0 + } + + # --- sync.json and the sftp batch ------------------------------------------ + $computer = ($env:COMPUTERNAME -replace '[^A-Za-z0-9-]', '') + if (-not $computer) { $computer = 'pc' } + $batch = '{0}-{1}' -f (Get-Date -Format 'yyyyMMdd-HHmmss'), $computer + $tempRoot = $env:TEMP + if (-not $tempRoot) { $tempRoot = [System.IO.Path]::GetTempPath() } + $work = Join-Path (Join-Path $tempRoot 'Sync-BPRT') $batch + New-Item -ItemType Directory -Path $work -Force | Out-Null + + $fileRows = @($local | ForEach-Object { [ordered]@{ path = $_.Path; size = $_.Size; sha256 = $_.Sha } }) + $sync = [ordered]@{ + format = 1 + tool = 'Sync-BPRT.ps1' + source = $Source + computer = $env:COMPUTERNAME + user = $env:USERNAME + created = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + manifest_generated = $(if ($manifest) { $manifest.generated } else { '' }) + files = $fileRows + uploaded = @($changed | ForEach-Object { $_.Path }) + removed = $removed + parse_errors = $problems.ToArray() # @() of an empty List fails: "Argument types do not match" + } + $syncPath = Join-Path $work 'sync.json' + Write-Utf8NoBom $syncPath ($sync | ConvertTo-Json -Depth 6) + + if ($ZipOut) { + Write-Step "Writing $ZipOut" + Add-Type -AssemblyName System.IO.Compression + Add-Type -AssemblyName System.IO.Compression.FileSystem + $zipFull = [System.IO.Path]::GetFullPath($ZipOut) + if (Test-Path -LiteralPath $zipFull) { Remove-Item -LiteralPath $zipFull -Force } + $zip = [System.IO.Compression.ZipFile]::Open($zipFull, [System.IO.Compression.ZipArchiveMode]::Create) + try { + foreach ($f in $changed) { + Write-Host " + $($f.Path)" + # Packages and zips do not compress; storing them saves minutes. + $level = [System.IO.Compression.CompressionLevel]::Optimal + if ($f.Path -match '\.(ppkg|zip|exe|png)$') { $level = [System.IO.Compression.CompressionLevel]::NoCompression } + [void][System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $f.Full, $f.Path, $level) + } + [void][System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $syncPath, 'sync.json', [System.IO.Compression.CompressionLevel]::Optimal) + } finally { + $zip.Dispose() + } + Write-Host '' + Write-Host ("Wrote {0} ({1})." -f $zipFull, (Format-Size (Get-Item -LiteralPath $zipFull).Length)) -ForegroundColor Green + Write-Host "Upload it under 'Upload a zip instead' on $finishUrl" + exit 0 + } + + $remote = "$($incoming.TrimEnd('/'))/$batch" + $part = "$remote.part" + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add('progress') # batch mode starts with the meter off + $lines.Add('-mkdir ' + (ConvertTo-SftpPath $incoming)) # leading '-': an existing folder is fine + $lines.Add('mkdir ' + (ConvertTo-SftpPath $part)) + $dirs = @($changed | ForEach-Object { $d = [System.IO.Path]::GetDirectoryName($_.Path); if ($d) { $d -replace '\\', '/' } } | Sort-Object -Unique) + $made = @{} + foreach ($d in $dirs) { + $acc = '' + foreach ($piece in ($d -split '/')) { + if ($acc) { $acc = "$acc/$piece" } else { $acc = $piece } + if (-not $made.ContainsKey($acc)) { $made[$acc] = $true; $lines.Add('-mkdir ' + (ConvertTo-SftpPath "$part/$acc")) } + } + } + foreach ($f in $changed) { + $lines.Add('put ' + (ConvertTo-SftpPath $f.Full) + ' ' + (ConvertTo-SftpPath "$part/$($f.Path)")) + } + # sync.json last, then the rename: the server only treats a folder without + # .part as a finished upload. + $lines.Add('put ' + (ConvertTo-SftpPath $syncPath) + ' ' + (ConvertTo-SftpPath "$part/sync.json")) + $lines.Add('rename ' + (ConvertTo-SftpPath $part) + ' ' + (ConvertTo-SftpPath $remote)) + $batchFile = Join-Path $work 'sftp-batch.txt' + Write-Utf8NoBom $batchFile (($lines -join "`r`n") + "`r`n") + + # BatchMode=no before -b: sftp -b turns BatchMode on (no password prompt), + # and the first value given wins. + $sftpArgs = @('-o', 'BatchMode=no', '-b', $batchFile, "$SshUser@$SshHost") + Write-Step 'Upload' + Write-Host " sync.json: $syncPath" + Write-Host " batch file: $batchFile" + Write-Host " command: sftp $($sftpArgs -join ' ')" + Write-Host (" uploads {0} file(s), {1}, into {2}" -f $changed.Count, (Format-Size $uploadBytes), $remote) + + if ($DryRun) { + Write-Host '' + Write-Host 'Dry run: nothing uploaded. Run again without -DryRun to upload.' -ForegroundColor Yellow + exit 0 + } + + $sftp = Get-Command sftp.exe -ErrorAction SilentlyContinue + if (-not $sftp) { $sftp = Get-Command sftp -ErrorAction SilentlyContinue } + if (-not $sftp) { + throw 'sftp.exe not found (Settings > Apps > Optional features > OpenSSH Client). Or use -ZipOut and upload the zip on the webapp page.' + } + Write-Host '' + Write-Host "Enter the password for $SshUser@$SshHost when asked." -ForegroundColor Cyan + & $sftp.Path @sftpArgs + if ($LASTEXITCODE -ne 0) { + throw "sftp failed (exit $LASTEXITCODE). The partial upload is in $part on the server; delete it on the webapp page, then run this again." + } + + Write-Host '' + Write-Host "Uploaded batch $batch." -ForegroundColor Green + Write-Host 'Validate and activate it here:' + Write-Host (" {0}batch/{1}" -f ($finishUrl.TrimEnd('/') + '/'), $batch) + exit 0 +} catch { + Write-Host '' + Write-Host "ERROR: $($_.Exception.Message)" -ForegroundColor Red + Write-Host $_.ScriptStackTrace -ForegroundColor DarkGray + exit 1 +} diff --git a/webapp/app.py b/webapp/app.py index 5f67604..e39cc8e 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -10,6 +10,10 @@ This file is the route surface; most logic lives in ``services/``: services.deploy - import_deploy + merge_tree + symlink dance services.unattend - parse + build + form-extract for unattend.xml services.wim - boot.wim startnet.cmd extract/update via wimtools + services.image_pages - per-image page tabs, header + overview data + services.control_files - preview/backup/atomic write/undo of control files + +The Updates pages are blueprints in ``views/`` (registered below). """ import hashlib @@ -35,7 +39,6 @@ from flask import ( stream_with_context, url_for, ) -from lxml import etree from werkzeug.utils import secure_filename import config @@ -43,17 +46,29 @@ from services import ( blancco_evidence, hardware_coverage, blancco_report, + dell_updates, deploy, + driver_map, fs, + control_files, + image_pages, image_registry, images, imaging_status, + mcl_ppkg, system, unattend, wim, ) from services.audit import audit from services.csrf import init_csrf +from views import blancco_update as blancco_update_views +from views import bprt_sync as bprt_sync_views +from views import dell_updates as dell_updates_views +from views import images as images_views +from views import images_drivers as images_drivers_views +from views import images_unattend as images_unattend_views +from views import ipxe as ipxe_views app = Flask(__name__) app.secret_key = config.FLASK_SECRET_KEY @@ -65,6 +80,21 @@ init_csrf(app) # baked-in config.py defaults on first run). Mutates config.* in place. image_registry.reload() +# Updates pages (Dell drivers/BIOS, iPXE, Blancco, BPRT sync): one blueprint +# each in views/, logic in services/; all routes live under /updates/. +app.register_blueprint(dell_updates_views.bp) +app.register_blueprint(ipxe_views.bp) +app.register_blueprint(blancco_update_views.bp) +app.register_blueprint(bprt_sync_views.bp) + +# GE Aerospace image pages: one page per image with tabs (Overview, Drivers, +# Unattend, OS and Packages) under templates/_image_header.html. The tab -> +# endpoint table is services/image_pages.IMAGE_TABS; unattend_editor below +# is the Unattend tab, the others are blueprints. +app.register_blueprint(images_views.bp) +app.register_blueprint(images_drivers_views.bp) +app.register_blueprint(images_unattend_views.bp) # Unattend review/save/undo + # --------------------------------------------------------------------------- # Routes - pages @@ -186,38 +216,15 @@ def unattend_editor(image_type): xml_file = fs.unattend_path(image_type) if request.method == "POST": + # Plain form post (the page itself saves through Review changes, + # views/images_unattend.py). Same path: the form is applied to the + # current text, backup, undo point, lint gate. save_mode = request.form.get("save_mode", "form") - if save_mode == "raw": - raw_xml = request.form.get("raw_xml", "") - try: - etree.fromstring(raw_xml.encode("utf-8")) - except etree.XMLSyntaxError as exc: - flash(f"Invalid XML: {exc}", "danger") - data = unattend.parse_unattend(xml_file) - data["raw_xml"] = raw_xml - return render_template( - "unattend_editor.html", - image_type=image_type, - friendly_name=config.FRIENDLY_NAMES.get(image_type, image_type), - data=data, - image_types=config.IMAGE_TYPES, - friendly_names=config.FRIENDLY_NAMES, - ) - xml_content = raw_xml + body = {"mode": "raw", "raw_xml": request.form.get("raw_xml", "")} else: - form_data = unattend.extract_form_data(request.form) - xml_content = unattend.build_unattend_xml(form_data) - - try: - os.makedirs(os.path.dirname(xml_file), exist_ok=True) - with open(xml_file, "w", encoding="utf-8") as fh: - fh.write(xml_content) - audit("UNATTEND_SAVE", f"{image_type} ({save_mode})") - flash("unattend.xml saved successfully.", "success") - except Exception as exc: - flash(f"Failed to save: {exc}", "danger") - + body = {"mode": "form", "form": unattend.extract_form_data(request.form)} + images_unattend_views.save_and_flash(image_type, body, f"Unattend save ({save_mode}, form post)") return redirect(url_for("unattend_editor", image_type=image_type)) data = unattend.parse_unattend(xml_file) @@ -225,77 +232,72 @@ def unattend_editor(image_type): "unattend_editor.html", image_type=image_type, friendly_name=config.FRIENDLY_NAMES.get(image_type, image_type), + active_tab="unattend", data=data, image_types=config.IMAGE_TYPES, friendly_names=config.FRIENDLY_NAMES, + **images_unattend_views.page_context(image_type), ) @app.route("/images//config") def image_config(image_type): + """Old single Configuration page, now split into tabs. Bookmarks land on + the Overview, or on one tab with ?tab=drivers|os|unattend.""" if image_type not in config.IMAGE_TYPES: flash("Unknown image type.", "danger") return redirect(url_for("dashboard")) + tab = request.args.get("tab", "overview") + if tab not in image_pages.TAB_IDS: + tab = "overview" + return redirect(image_pages.tab_url(image_type, tab)) - cfg = images.load_image_config(image_type) - return render_template( - "image_config.html", - image_type=image_type, - friendly_name=config.FRIENDLY_NAMES.get(image_type, image_type), - config=cfg, - ) + +# Legacy /config/save sections and the tab each one now lives on. +_CONFIG_SECTIONS = { + "hardware_models": "os", + "drivers": "drivers", + "operating_systems": "os", + "packages": "os", +} @app.route("/images//config/save", methods=["POST"]) def image_config_save(image_type): + """Old one-section form save. No page posts here any more, and it must + not write: it would skip the diff preview, the stale-file check and the + Drivers tab's lint gate. It sends the user to the tab that saves now.""" if image_type not in config.IMAGE_TYPES: flash("Unknown image type.", "danger") return redirect(url_for("dashboard")) + tab = _CONFIG_SECTIONS.get(request.form.get("section", ""), "overview") + flash("This form no longer saves. Make the change on this tab and use Review and save; " + "nothing was written.", "warning") + return redirect(image_pages.tab_url(image_type, tab)) - section = request.form.get("section", "") - payload = request.form.get("payload", "[]") - try: - data = json.loads(payload) - except json.JSONDecodeError: - flash("Invalid JSON payload.", "danger") - return redirect(url_for("image_config", image_type=image_type)) - ctrl = fs.control_path(image_type) - tools = fs.tools_path(image_type) - - try: - if section == "hardware_models": - us_file = os.path.join(tools, "user_selections.json") - us_raw = fs.load_json(us_file) - us_data = us_raw[0] if us_raw and isinstance(us_raw, list) else {} - us_data["HardwareModelSelection"] = data - fs.save_json(us_file, [us_data]) - audit("CONFIG_SAVE", f"{image_type}/hardware_models") - - elif section == "drivers": - clean = [{k: v for k, v in d.items() if not k.startswith("_")} for d in data] - fs.save_json(os.path.join(ctrl, "HardwareDriver.json"), clean) - audit("CONFIG_SAVE", f"{image_type}/drivers") - - elif section == "operating_systems": - clean = [{k: v for k, v in d.items() if not k.startswith("_")} for d in data] - fs.save_json(os.path.join(ctrl, "OperatingSystem.json"), clean) - audit("CONFIG_SAVE", f"{image_type}/operating_systems") - - elif section == "packages": - clean = [{k: v for k, v in d.items() if not k.startswith("_")} for d in data] - fs.save_json(os.path.join(ctrl, "packages.json"), clean) - audit("CONFIG_SAVE", f"{image_type}/packages") - - else: - flash(f"Unknown section: {section}", "danger") - return redirect(url_for("image_config", image_type=image_type)) - - flash(f"Saved {section.replace('_', ' ')} successfully.", "success") - except Exception as exc: - flash(f"Failed to save {section}: {exc}", "danger") - - return redirect(url_for("image_config", image_type=image_type)) +def _register_package(image_type, fname, dest): + """Append {fileName, destinationDir} to packages.json through the same + path as the OS and Packages tab (format kept, .bak--webui backup, + atomic, the tab's undo point). False when it is already listed.""" + path = image_pages.control_file(image_type, "packages.json") + text = control_files.read_text(path) + entries = [] + if text.strip(): + try: + entries = json.loads(text.lstrip("\ufeff")) + except ValueError: + entries = None + if not isinstance(entries, list): + raise ValueError("packages.json is not a JSON list, so the file was not registered") + if any(str(e.get("fileName") or e.get("FileName") or "").lower() == fname.lower() + for e in entries if isinstance(e, dict)): + return False + entries.append({"fileName": fname, "destinationDir": dest}) + plan = control_files.plan_json(path, entries, "packages.json") + control_files.commit(image_type, images_views.UNDO_SCOPE, [plan], + action=f"Package upload: {fname} registered") + return True # --------------------------------------------------------------------------- @@ -304,71 +306,65 @@ def image_config_save(image_type): @app.route("/images//drivers/upload", methods=["POST"]) def images_drivers_upload(image_type): + """Store a driver .zip under Out-of-box Drivers (services/driver_map. + save_upload: temp file + rename, in-use zips never overwritten). It is + NOT added to HardwareDriver.json here: the Drivers tab's Add to map does + that through the review step, with a backup and Undo. family and + destination_dir from old forms are ignored for that reason.""" if image_type not in config.IMAGE_TYPES: flash(f"Unknown image type: {image_type}", "danger") return redirect(url_for("dashboard")) + back = redirect(image_pages.tab_url(image_type, "drivers")) f = request.files.get("driver_file") if not f or not f.filename: flash("No file selected.", "danger") - return redirect(url_for("image_config", image_type=image_type)) - family = (request.form.get("family") or "").strip() - dest = (request.form.get("destination_dir") or "").strip() + return back overwrite = request.form.get("overwrite") == "1" try: - rec = images.upload_driver(image_type, f, family=family, - destination_dir=dest, overwrite=overwrite) - audit("DRIVER_UPLOAD", f"{image_type}/{rec['filename']} registered={rec['registered']}") - msg = f"Uploaded {rec['filename']}." - msg += " Registered to family in HardwareDriver.json." if rec["registered"] else " Now in orphans list until adopted." - flash(msg, "success") - except (ValueError, FileExistsError, FileNotFoundError) as ex: + rec = driver_map.save_upload(image_type, f, overwrite=overwrite) + audit("DRIVER_UPLOAD", f"{image_type}/{rec['filename']} replaced={rec['replaced']}") + flash(f"Uploaded {rec['filename']}. It is listed under Unregistered zips; " + "use Add to map there to give it an entry.", "success") + except (ValueError, FileExistsError) as ex: flash(str(ex), "danger") except Exception as ex: flash(f"Upload failed: {ex}", "danger") - return redirect(url_for("image_config", image_type=image_type)) + return back @app.route("/images//drivers/adopt", methods=["POST"]) def images_drivers_adopt(image_type): + """Old one-click adopt. It rewrote HardwareDriver.json with indent 2, no + backup and an entry with no modelswminame (PESetup throws on that), so + it now writes nothing and points at Add to map on the Drivers tab.""" if image_type not in config.IMAGE_TYPES: flash(f"Unknown image type: {image_type}", "danger") return redirect(url_for("dashboard")) - filename = (request.form.get("filename") or "").strip() - family = (request.form.get("family") or "").strip() - dest = (request.form.get("destination_dir") or "").strip() - try: - rec = images.adopt_orphan(image_type, filename, family, dest) - audit("DRIVER_ADOPT", f"{image_type}/{rec['filename']} family={family}") - if rec.get("already_registered"): - flash(f"{rec['filename']} was already registered.", "info") - else: - flash(f"Adopted {rec['filename']} into HardwareDriver.json.", "success") - except (ValueError, FileNotFoundError) as ex: - flash(str(ex), "danger") - except Exception as ex: - flash(f"Adopt failed: {ex}", "danger") - return redirect(url_for("image_config", image_type=image_type)) + flash("Use Add to map under Unregistered zips: the new entry needs model tokens and is " + "saved through Review and save. Nothing was written.", "info") + return redirect(image_pages.tab_url(image_type, "drivers")) @app.route("/images//drivers/orphans/delete", methods=["POST"]) def images_drivers_orphans_delete(image_type): + """Old orphan delete (form field "filename"), now the same safe delete + as the Drivers tab: zips any image, undo or rollback uses are kept.""" if image_type not in config.IMAGE_TYPES: flash(f"Unknown image type: {image_type}", "danger") return redirect(url_for("dashboard")) - # Filenames come as repeated form fields (one per checkbox). + back = redirect(image_pages.tab_url(image_type, "drivers")) filenames = request.form.getlist("filename") if not filenames: flash("No files selected for removal.", "warning") - return redirect(url_for("image_config", image_type=image_type)) - rec = images.remove_orphans(image_type, filenames) - audit("DRIVER_ORPHAN_REMOVE", f"{image_type} removed={len(rec['removed'])} missing={len(rec['missing'])}") - parts = [] + return back + rec = driver_map.delete_orphans(image_type, filenames) + audit("DRIVER_ORPHAN_REMOVE", f"{image_type} removed={len(rec['removed'])} " + f"refused={len(rec['refused'])} {', '.join(rec['removed'])}") if rec["removed"]: - parts.append(f"Removed {len(rec['removed'])} orphan(s).") - if rec["missing"]: - parts.append(f"{len(rec['missing'])} not found / errored.") - flash(" ".join(parts) or "No-op.", "success" if rec["removed"] else "warning") - return redirect(url_for("image_config", image_type=image_type)) + flash(f"Deleted {len(rec['removed'])} unregistered zip(s).", "success") + for rel, why in rec["refused"]: + flash(f"Not deleted: {rel} ({why}).", "warning") + return back @app.route("/images//packages/upload", methods=["POST"]) @@ -379,20 +375,31 @@ def images_packages_upload(image_type): f = request.files.get("package_file") if not f or not f.filename: flash("No file selected.", "danger") - return redirect(url_for("image_config", image_type=image_type)) + return redirect(image_pages.tab_url(image_type, "os")) dest = (request.form.get("destination_dir") or "").strip() overwrite = request.form.get("overwrite") == "1" try: - rec = images.upload_package(image_type, f, destination_dir=dest, overwrite=overwrite) - audit("PACKAGE_UPLOAD", f"{image_type}/{rec['filename']} registered={rec['registered']}") + # The file only; packages.json is edited here through control_files + # (services.images.upload_package would rewrite it without a backup). + rec = images.upload_package(image_type, f, destination_dir="", overwrite=overwrite) msg = f"Uploaded {rec['filename']}." - msg += " Registered in packages.json." if rec["registered"] else " Not registered (no destination_dir provided)." + registered = False + if dest: + try: + registered = _register_package(image_type, rec["filename"], dest) + msg += (" Registered in packages.json (backup made; Undo last save puts it back)." + if registered else " Already listed in packages.json.") + except (ValueError, OSError) as ex: + flash(f"Not registered: {ex}", "warning") + else: + msg += " Not registered (no destination_dir provided)." + audit("PACKAGE_UPLOAD", f"{image_type}/{rec['filename']} registered={registered}") flash(msg, "success") except (ValueError, FileExistsError) as ex: flash(str(ex), "danger") except Exception as ex: flash(f"Upload failed: {ex}", "danger") - return redirect(url_for("image_config", image_type=image_type)) + return redirect(image_pages.tab_url(image_type, "os")) # --------------------------------------------------------------------------- @@ -761,8 +768,13 @@ def hardware_seen(): @app.route("/hardware") def hardware_page(): """Every model WinPE has seen, with driver-pack and BIOS coverage.""" + try: + # Files with a newer Dell pack/BIOS in the last Updates > Dell scan. + hints = dell_updates.update_hints() + except Exception: + hints = {"drivers": {}, "bios": {}} return render_template("hardware.html", rows=hardware_coverage.coverage(), - images=hardware_coverage.IMAGES) + images=hardware_coverage.IMAGES, hints=hints) @app.route("/hardware/dismiss", methods=["POST"]) @@ -1048,6 +1060,9 @@ def enrollment(): return render_template( "enrollment.html", packages=packages, + mcl_packages=mcl_ppkg.packages(), + mcl_archive=mcl_ppkg.archived(), + mcl_links=mcl_ppkg.image_links(), active_ppkg=read_ppkg_conf(), image_types=config.IMAGE_TYPES, friendly_names=config.FRIENDLY_NAMES, @@ -1069,6 +1084,13 @@ def enrollment_upload(): if not filename.lower().endswith(".ppkg"): flash("Only .ppkg files are accepted.", "danger") return redirect(url_for("enrollment")) + # An MCL name would also match the ppkg.conf regex below and become the + # shopfloor package; those go to BPRT through the MCL section instead. + # Check the sanitised name too: secure_filename turns spaces into + # underscores, which can make an MCL name out of one that was not. + if mcl_ppkg.is_mcl_name(f.filename) or mcl_ppkg.is_mcl_name(filename): + flash(f"{filename} is a standard/engineer MCL package; upload it in the MCL section.", "danger") + return redirect(url_for("enrollment") + "#mcl") os.makedirs(config.ENROLLMENT_PPKG_DIR, exist_ok=True) dest = os.path.join(config.ENROLLMENT_PPKG_DIR, filename) @@ -1093,6 +1115,56 @@ def enrollment_upload(): return redirect(url_for("enrollment")) +@app.route("/enrollment/mcl/upload", methods=["POST"]) +def mcl_ppkg_upload(): + """Standard/engineer MCL package: replaces the one with the same + tenant/environment/purpose, archiving the old (services.mcl_ppkg).""" + f = request.files.get("ppkg_file") + if not f or not f.filename: + flash("No file selected.", "danger") + return redirect(url_for("enrollment") + "#mcl") + try: + info, moved = mcl_ppkg.install_upload(f, f.filename) + except (ValueError, OSError) as exc: + flash(f"Upload rejected: {exc}", "danger") + return redirect(url_for("enrollment") + "#mcl") + name = info["filename"] + audit("MCL_PPKG_UPLOAD", f"{name} replaces {', '.join(moved) or 'nothing'}") + flash(f"Installed {name} for {info['prefix']}" + + (f"; archived {', '.join(moved)}." if moved else "."), "success") + for warning in info["warnings"]: + flash(warning + ".", "warning") + return redirect(url_for("enrollment") + "#mcl") + + +@app.route("/enrollment/mcl/archive", methods=["POST"]) +def mcl_ppkg_archive(): + """Move one package out of BPRT (clears a duplicate made by hand).""" + try: + name = mcl_ppkg.archive(request.form.get("filename", "")) + except (ValueError, OSError) as exc: + flash(f"Archive failed: {exc}", "danger") + return redirect(url_for("enrollment") + "#mcl") + audit("MCL_PPKG_ARCHIVE", name) + flash(f"Archived {name}.", "success") + return redirect(url_for("enrollment") + "#mcl") + + +@app.route("/enrollment/mcl/restore", methods=["POST"]) +def mcl_ppkg_restore(): + batch = request.form.get("batch", "") + name = request.form.get("filename", "") + try: + info, moved = mcl_ppkg.restore(batch, name) + except (ValueError, OSError) as exc: + flash(f"Restore failed: {exc}", "danger") + return redirect(url_for("enrollment") + "#mcl") + name = info["filename"] + audit("MCL_PPKG_RESTORE", f"{batch}/{name} replaces {', '.join(moved) or 'nothing'}") + flash(f"Restored {name}" + (f"; archived {', '.join(moved)}." if moved else "."), "success") + return redirect(url_for("enrollment") + "#mcl") + + @app.route("/enrollment/download/") def enrollment_download(filename): filename = secure_filename(filename) @@ -1492,31 +1564,24 @@ def api_save_unattend(image_type): xml_file = fs.unattend_path(image_type) payload = request.get_json(silent=True) - if not payload: + if not payload or not isinstance(payload, dict): return jsonify({"error": "No JSON body provided"}), 400 + # {"raw_xml": ...} writes the text as is; any other body is form data + # (keys left out stay as they are in the file). Backup, undo point and + # lint gate as on the Unattend tab. if "raw_xml" in payload: - raw_xml = payload["raw_xml"] - try: - etree.fromstring(raw_xml.encode("utf-8")) - except etree.XMLSyntaxError as exc: - return jsonify({"error": f"Invalid XML: {exc}"}), 400 - xml_content = raw_xml + body = {"mode": "raw", "raw_xml": payload["raw_xml"]} else: - try: - xml_content = unattend.build_unattend_xml(payload) - except Exception as exc: - return jsonify({"error": f"Failed to build XML: {exc}"}), 400 - + body = {"mode": "form", "form": payload} try: - os.makedirs(os.path.dirname(xml_file), exist_ok=True) - with open(xml_file, "w", encoding="utf-8") as fh: - fh.write(xml_content) - except Exception as exc: - return jsonify({"error": f"Failed to write file: {exc}"}), 500 + plan = images_unattend_views.save(image_type, body, action="Unattend save (API)") + except images_unattend_views.Refused as exc: + return jsonify({"error": str(exc)}), exc.status - audit("UNATTEND_SAVE_API", image_type) - return jsonify({"status": "ok", "path": xml_file}) + if plan["changed"]: + audit("UNATTEND_SAVE_API", image_type) + return jsonify({"status": "ok", "path": xml_file, "changed": plan["changed"]}) # --------------------------------------------------------------------------- @@ -1559,11 +1624,24 @@ def timestamp_fmt(ts): return local_dt(ts) +@app.template_filter("tojson_ordered") +def tojson_ordered(value): + """JSON in the data's own key order (Flask's tojson sorts keys, which + turns a save of an unchanged row into a reordered-keys diff). Output is + a plain string, so it is HTML-escaped like any other value.""" + return json.dumps(value, ensure_ascii=False) + + @app.context_processor def inject_globals(): return { "all_image_types": config.IMAGE_TYPES, "all_friendly_names": config.FRIENDLY_NAMES, + # image pages: tab table, header data, tab URLs (see _image_header.html) + "image_tabs": image_pages.IMAGE_TABS, + "image_header_info": image_pages.header_info, + "image_tab_url": image_pages.tab_url, + "image_tab_for_endpoint": image_pages.tab_for_endpoint, "blancco_todo": dict(blancco_evidence.todo_counts(), hardware=hardware_coverage.gap_count(), total=blancco_evidence.todo_counts()["total"] + hardware_coverage.gap_count()), diff --git a/webapp/config.py b/webapp/config.py index 84b2800..7640bcf 100644 --- a/webapp/config.py +++ b/webapp/config.py @@ -32,6 +32,13 @@ DISPLAY_TZ = os.environ.get("DISPLAY_TZ", "America/New_York") HARDWARE_STATE = os.environ.get("HARDWARE_STATE", "/var/lib/pxe-webapp/hardware-seen.json") BIOS_DIR = os.environ.get("BIOS_DIR", "/srv/samba/winpeapps/_shared/BIOS") BLANCCO_STATE = os.environ.get("BLANCCO_STATE", "/var/lib/pxe-webapp/blancco-state.json") +# PXE server address the Blancco preferences must point the report share at +# (blancco-init.sh greps for it too); checked when a release is staged. +BLANCCO_SERVER = os.environ.get("BLANCCO_SERVER", "172.16.9.1") +# Updates pages (Dell drivers/BIOS, iPXE): settings, staged downloads, backups. +UPDATES_DIR = os.environ.get("UPDATES_DIR", "/var/lib/pxe-webapp/updates") +TFTP_ROOT = os.environ.get("TFTP_ROOT", "/srv/tftp") +DNSMASQ_CONF = os.environ.get("DNSMASQ_CONF", "/etc/dnsmasq.conf") # Log sources used by services/imaging_log_tail.py to infer client progress # when no /imaging/status push has arrived yet. Override per-host if any of diff --git a/webapp/services/blancco_update.py b/webapp/services/blancco_update.py new file mode 100644 index 0000000..95cbdce --- /dev/null +++ b/webapp/services/blancco_update.py @@ -0,0 +1,1459 @@ +"""Install Blancco Drive Eraser releases from the webapp, test before switch. + +A release is a vendor ISO (or the vendor's USB image, .img or .img.gz, with +the ISO inside its FAT32 partition). What it replaces in WEB_ROOT/blancco: + + * arch/x86_64/airootfs.sfs - the rootfs; both boot paths fetch it over HTTP + * vmlinuz-bde-linux, - Blancco's own kernel + initramfs, used by the + initramfs-bde-linux.img BIOS (native) path; the initramfs is patched + so copytoram keeps the network up + * intel-ucode.img, amd-ucode.img + * config.img - vendor cpio (report key, licence container, + certificates) with OUR preferences written in + * preferences.xml - our preferences, served for the UEFI path + * config-clean.xml - config.xml from config.img, NULs stripped + (the licence container the UEFI path fetches) + * blancco-shim.efi - Blancco's Microsoft-signed shim from the ISO + (EFI/BOOT/bootx64.efi), when the ISO has one + +The UEFI path's Ubuntu kernel (vmlinuz-ubuntu*), its initrd (kexec-initrd*) +and kmod tarballs, grubx64.efi, shimx64.efi and blancco-progress.py are NOT +part of a release and are never touched here. + +The transformations are the ones scripts/prepare-boot-tools.sh does (same +checks, same results); keep the two in step. The preferences guard is the +important one: our preferences are a merge onto one vendor release's +preferences, and a release whose generation.major differs would +silently lose settings the new schema added or renamed. Staging STOPS there +until a human merges playbook/blancco-preferences.xml onto the new vendor +file and uploads the result. + +Flow: + stage (background job, python3 -m services.blancco_update stage FILE): + extract into WEB_ROOT/blancco/staged//, HTTP-served so a + test menu entry can boot it; + test a menu entry in GetPxeScript.aspx boots the staged files (BIOS: + Blancco's kernel with archiso_http_srv at the staged dir; UEFI: + the live Ubuntu kernel line plus blancco.base=, which needs the + init from playbook/blancco-init.sh as of 2026-10-13); + activate back up the current release files to + WEB_ROOT/blancco/backup--/ (hard links) and swap each + staged file in with an atomic rename; + rollback the same swap from any backup, after backing up what is live. +""" +from __future__ import annotations + +import gzip +import lzma +import os +import re +import shutil +import signal +import struct +import subprocess +import sys +import tempfile +import threading +import time +import xml.etree.ElementTree as ET +from datetime import datetime + +import config +from services import updates_common as uc + +JOB_UNIT = "pxe-webapp-blancco-stage" +# The PXE server address the preferences must point the report share at +# (blancco-init.sh greps for it too). +SERVER = config.BLANCCO_SERVER + +# Release path (relative to WEB_ROOT/blancco) <- member of the vendor ISO. +ISO_MEMBERS = { + "vmlinuz-bde-linux": "arch/boot/x86_64/vmlinuz-bde-linux", + "initramfs-bde-linux.img": "arch/boot/x86_64/initramfs-bde-linux.img", + "intel-ucode.img": "arch/boot/intel-ucode.img", + "amd-ucode.img": "arch/boot/amd-ucode.img", + "config.img": "arch/boot/config.img", + "arch/x86_64/airootfs.sfs": "arch/x86_64/airootfs.sfs", +} +ISO_OPTIONAL = { + "blancco-shim.efi": "EFI/BOOT/bootx64.efi", +} +ISO_VERSION = "arch/version" +ISO_SHA512 = "arch/x86_64/airootfs.sha512" +# Every file a release must have, in swap order: the rootfs first, the +# small files the boot reads first last, so a boot during the swap mostly +# sees one release. +RELEASE_FILES = ( + "arch/x86_64/airootfs.sfs", + "vmlinuz-bde-linux", + "initramfs-bde-linux.img", + "intel-ucode.img", + "amd-ucode.img", + "config-clean.xml", + "preferences.xml", + "config.img", +) +OPTIONAL_FILES = ("blancco-shim.efi",) +# Kernel file names in the boot menu that a test entry rewrites to the staged +# copy (the Ubuntu kernel and its initrd stay where they are). +_MENU_RELEASE_NAMES = ("vmlinuz-bde-linux", "initramfs-bde-linux.img", + "intel-ucode.img", "amd-ucode.img", "config.img") +SOURCE_SUFFIXES = (".iso", ".img", ".img.gz", ".iso.gz") +_SAFE_VERSION = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._-]{0,39}$") +_SAFE_NAME = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._ ()+-]{0,199}$") + +TEST_LABEL = "blancco_staged" +TEST_BEGIN = "# BEGIN blancco-staged test entry (PXE webapp, Updates > Blancco)" +TEST_END = "# END blancco-staged test entry" + +# Serialises the inline swaps (activate, rollback, menu edits, prefs merge). +_lock = threading.Lock() + + +class StageBlocked(Exception): + """Staging cannot finish until a human acts (preferences merge).""" + + +# --- paths ------------------------------------------------------------------ + +def blancco_root(): + return os.path.join(config.WEB_ROOT, "blancco") + + +def staged_root(): + return os.path.join(blancco_root(), "staged") + + +def _dir(*parts): + return uc.state_dir("blancco", *parts) + + +def upload_dir(): + return _dir("uploads") + + +def job_file(): + return os.path.join(_dir(), "job.json") + + +def menu_path(): + return os.path.join(config.WEB_ROOT, "Altiris", "iPXE", "GetPxeScript.aspx") + + +def _vendor_dir(version): + return os.path.join(_dir("vendor"), version) + + +def merged_prefs_path(version): + return os.path.join(_dir("prefs"), f"{version}.xml") + + +def active_prefs_path(): + """Our preferences as served now (the playbook deploys + playbook/blancco-preferences.xml here).""" + return os.path.join(blancco_root(), "preferences.xml") + + +def _check_version(version): + if not _SAFE_VERSION.match(str(version or "")): + raise ValueError(f"bad version name: {version!r}") + return version + + +def _ts(): + return datetime.now().strftime("%Y%m%d-%H%M%S") + + +# --- reading release files -------------------------------------------------- + +def kernel_version(path): + """Version string from a bzImage header ('6.12.93-1bde-linux'), or None.""" + try: + with open(path, "rb") as f: + head = f.read(0x10000) + except OSError: + return None + if head[0x202:0x206] != b"HdrS": + return None + off = struct.unpack_from("//, or None.""" + m = re.search(rb"\s*(\d+)\s*(\d+)" + rb"(?:\s*(\d+))?", xml or b"") + if not m: + return None + return ".".join(x.decode() for x in m.groups() if x is not None) + + +def _guard_version(xml): + # Exactly what prepare-boot-tools.sh compares: generation + major. + m = re.search(rb"\s*(\d+)\s*" + rb"(\d+)", xml) + return m.groups() if m else None + + +def _read(path, limit=None): + try: + with open(path, "rb") as f: + return f.read() if limit is None else f.read(limit) + except OSError: + return None + + +def release_info(folder): + """What is in a release folder (the live one, a staged one or a backup). + release.json, written by this module, wins; anything missing is read from + the files themselves. A release.json whose file fingerprint no longer + matches (files copied over by hand or by a playbook run) is ignored.""" + meta = uc.read_json(os.path.join(folder, "release.json"), {}) or {} + if meta.get("fingerprint") and meta["fingerprint"] != _fingerprint(folder): + meta = {"changed_by_hand": True} + info = dict(meta) + files, total, missing = {}, 0, [] + for rel in RELEASE_FILES + OPTIONAL_FILES: + path = _find_release_file(folder, rel) + if not path: + if rel in RELEASE_FILES: + missing.append(rel) + continue + st = os.stat(path) + files[rel] = {"size": st.st_size, "mtime": st.st_mtime} + total += st.st_size + info["files"] = files + info["missing"] = missing + info["size"] = total + if not info.get("kernel"): + info["kernel"] = kernel_version(os.path.join(folder, "vmlinuz-bde-linux")) + if not info.get("prefs_version"): + cfg = _read(os.path.join(folder, "config.img")) + info["prefs_version"] = prefs_version(cpio_member(cfg or b"", "preferences.xml")) + if not info.get("version"): + # A release installed by hand: the vendor's preferences version is + # the Blancco version (7.20.0 = Drive Eraser 7.20), and ours is + # merged onto it, so it still says which release is in config.img. + info["version"] = info.get("prefs_version") or "unknown" + info["inferred"] = True + rootfs = files.get("arch/x86_64/airootfs.sfs") + info["rootfs_mtime"] = rootfs["mtime"] if rootfs else None + return info + + +def _find_release_file(folder, rel): + path = os.path.join(folder, rel) + if os.path.isfile(path): + return path + # Hand-made backups keep airootfs.sfs flat. + flat = os.path.join(folder, os.path.basename(rel)) + if rel != os.path.basename(rel) and os.path.isfile(flat): + return flat + return None + + +def _fingerprint(folder): + """Size + mtime of each release file: tells release.json whether the + files are still the ones it describes (an in-place cp changes both).""" + out = {} + for rel in RELEASE_FILES + OPTIONAL_FILES: + path = _find_release_file(folder, rel) + if path: + st = os.stat(path) + out[rel] = [st.st_size, int(st.st_mtime)] + return out + + +def current(): + info = release_info(blancco_root()) + info["path"] = blancco_root() + return info + + +def staged(): + out = [] + root = staged_root() + if not os.path.isdir(root): + return out + test = test_entry() + for name in sorted(os.listdir(root)): + path = os.path.join(root, name) + if name.startswith(".") or not os.path.isdir(path) or not _SAFE_VERSION.match(name): + continue + info = release_info(path) + info["name"] = name + info["status"] = info.get("status") or "unknown" + info["in_menu"] = bool(test and test.get("version") == name) + info["merged_prefs"] = os.path.isfile(merged_prefs_path(name)) + out.append(info) + return out + + +def backups(): + out = [] + root = blancco_root() + if not os.path.isdir(root): + return out + for name in os.listdir(root): + path = os.path.join(root, name) + if not name.startswith("backup-") or not os.path.isdir(path): + continue + info = release_info(path) + info["name"] = name + info["mtime"] = os.stat(path).st_mtime + info["created"] = info.get("backed_up") or info["mtime"] + info["complete"] = not info["missing"] + out.append(info) + out.sort(key=lambda b: b["mtime"], reverse=True) + return out + + +def source_dirs(): + """Folders a release file may be picked from: our upload folder, and the + image upload folder (files copied in over SMB or scp).""" + dirs = [("uploads", upload_dir())] + if getattr(config, "UPLOAD_DIR", None): + dirs.append(("image-upload", config.UPLOAD_DIR)) + return dirs + + +def sources(): + out = [] + for key, folder in source_dirs(): + try: + names = os.listdir(folder) + except OSError: + continue + for name in sorted(names): + path = os.path.join(folder, name) + if not name.lower().endswith(SOURCE_SUFFIXES) or not os.path.isfile(path): + continue + st = os.stat(path) + out.append({"key": key, "name": name, "path": path, "size": st.st_size, + "modified": st.st_mtime, "deletable": key == "uploads"}) + return out + + +def resolve_source(key, name): + for k, folder in source_dirs(): + if k == key: + if name != os.path.basename(name) or not name.lower().endswith(SOURCE_SUFFIXES): + break + path = os.path.join(folder, name) + if os.path.isfile(path): + return path + raise ValueError("no such release file") + + +def clean_upload_name(name): + name = os.path.basename(str(name or "").replace("\\", "/")).strip() + if not _SAFE_NAME.match(name) or not name.lower().endswith(SOURCE_SUFFIXES): + raise ValueError("upload a Blancco .iso, .img or .img.gz (letters, digits, . _ - only)") + return name + + +def save_upload(stream, name, length=None): + """Stream an upload to the upload folder (never held in memory). Written + to a .part file and renamed when complete.""" + name = clean_upload_name(name) + folder = upload_dir() + dest = os.path.join(folder, name) + if length: + free = shutil.disk_usage(folder).free + if length > free - (1 << 30): + raise RuntimeError(f"not enough disk space for {length // (1 << 20)} MB " + f"({free // (1 << 20)} MB free)") + # A part file of its own, so two uploads of the same name cannot + # interleave their blocks into one file. + fd, part = tempfile.mkstemp(dir=folder, prefix=f".{name}.", suffix=".part") + written = 0 + try: + with os.fdopen(fd, "wb") as f: + while True: + block = stream.read(1 << 20) + if not block: + break + f.write(block) + written += len(block) + if length and written != length: + raise RuntimeError(f"upload incomplete: {written} of {length} bytes") + if not written: + raise RuntimeError("empty upload") + os.chmod(part, 0o644) # mkstemp makes 0600 + os.replace(part, dest) + finally: + if os.path.exists(part): + os.remove(part) + return name, written + + +def delete_upload(name): + path = os.path.join(upload_dir(), clean_upload_name(name)) + if not os.path.isfile(path): + raise ValueError("no such upload") + os.remove(path) + + +# --- transformations (same as scripts/prepare-boot-tools.sh) ---------------- + +def check_prefs(ours, vendor, server=SERVER): + """Raise StageBlocked when our preferences are not a merge onto this + release's preferences, ValueError when they lack the report share.""" + if _guard_version(ours) != _guard_version(vendor): + raise StageBlocked( + f"Our preferences are for Blancco {prefs_version(ours) or '?'}, this release's " + f"vendor preferences are {prefs_version(vendor) or '?'}. The preferences format " + "can change between releases, so playbook/blancco-preferences.xml must be merged " + "onto the new vendor preferences.xml first (download it below), then upload the " + "merged file to finish staging.") + for need in (f"{server}".encode(), b"blancco-reports", + b"true"): + if need not in ours: + raise ValueError(f"our preferences are missing {need.decode()}") + + +def patch_preferences(vendor, ours, server=SERVER): + """Vendor preferences.xml (as stored in config.img) with our XML in place + of theirs, at exactly the vendor file's size. The vendor file is + [NUL padding] : the gap is refilled, or the + filler comment gives up bytes when ours is longer.""" + check_prefs(ours, vendor, server) + orig_size = len(vendor) + xml = ours.strip() + b"\n" + end = vendor.find(b"") + cstart = vendor.rfind(b"\n"): + raise ValueError(f"preferences need {cut} more bytes than the padding and " + "filler comment can give") + tail = tail[:5] + tail[5 + cut:] + gap = b"" + out = xml + gap + tail + if len(out) != orig_size: + raise ValueError(f"preferences.xml size changed {orig_size} to {len(out)}") + return out + + +def patch_config_img(data, ours, server=SERVER): + """config.img with our preferences written in. The file keeps its size, + so its bytes are replaced in place and every other member (report key, + licence container, certificates) and header stays the vendor's.""" + for e in cpio_entries(data): + if e["name"].lstrip("./") != "preferences.xml": + continue + start, size = e["data_off"], e["size"] + new = patch_preferences(data[start:start + size], ours, server) + out = bytearray(data) + out[start:start + size] = new + if data.startswith(b"070702", e["header"]): + # crc format: the header carries a byte sum of the file data. + check = b"%08X" % (sum(new) & 0xFFFFFFFF) + out[e["header"] + 102:e["header"] + 110] = check + return bytes(out) + raise ValueError("config.img has no preferences.xml") + + +def config_clean(config_img): + """config.xml from config.img with the NUL padding stripped (what the + playbook serves as config-clean.xml).""" + xml = cpio_member(config_img, "config.xml") + if xml is None: + raise ValueError("config.img has no config.xml") + return xml.replace(b"\x00", b"") + + +def patch_initramfs(data): + """Disable archiso_pxe_common's network flush after copytoram (BIOS path). + 7.20 prepends an uncompressed early cpio (NIC firmware) to the XZ + archive: walk its headers to TRAILER!!! and keep it byte for byte.""" + off = 0 + while data.startswith(b"070701", off): + namesize = int(data[off + 94:off + 102], 16) + filesize = int(data[off + 54:off + 62], 16) + name = data[off + 110:off + 110 + namesize - 1] + off = (((off + 110 + namesize + 3) & ~3) + filesize + 3) & ~3 + if name == b"TRAILER!!!": + while off < len(data) and data[off] == 0: + off += 1 + if not data.startswith(b"\xfd7zXZ\x00", off): + raise ValueError(f"initramfs layout not recognised (no XZ archive at {off})") + prefix, compressed = data[:off], data[off:] + raw = lzma.decompress(compressed) + old = b'"y" ]; then\n for curif in /sys/class/net' + new = b'"N" ]; then\n for curif in /sys/class/net' + count = raw.count(old) + if count != 1: + raise ValueError(f"copytoram network-flush pattern found {count} times, expected 1; " + "BIOS PXE boot would lose the network after copytoram") + raw = raw.replace(old, new, 1) + return prefix + lzma.compress(raw, format=lzma.FORMAT_XZ, check=lzma.CHECK_CRC32, preset=6) + + +# --- getting the ISO out of the upload --------------------------------------- + +def _open_source(path): + with open(path, "rb") as f: + magic = f.read(2) + return gzip.open(path, "rb") if magic == b"\x1f\x8b" else open(path, "rb") + + +def _is_iso(f): + f.seek(0x8001) + return f.read(5) == b"CD001" + + +class _Fat: + """Just enough FAT16/FAT32 to find one file and where its clusters are. + Works on a gzip stream: reads go forward, the file data in order.""" + + def __init__(self, f, base): + self.f, self.base = f, base + f.seek(base) + bs = f.read(512) + if len(bs) < 512 or bs[510:512] != b"\x55\xaa" or bs[0] not in (0xEB, 0xE9): + raise ValueError("no FAT boot sector") + (bps, spc, rsv, nfats, rootents, tot16, _media, + fsz16) = struct.unpack_from("= 65525 + self.root_cluster = struct.unpack_from(" 0: + raise ValueError("FAT cluster chain shorter than the file") + return trimmed + + def listdir(self, cluster=None): + if cluster is None and not self.fat32: + self.f.seek(self.root_region[0]) + raw = self.f.read(self.root_region[1]) + else: + raw = b"" + for off, length in self.runs(cluster or self.root_cluster, + len(self.chain(cluster or self.root_cluster)) * self.csize): + self.f.seek(off) + raw += self.f.read(length) + out, lfn = [], {} + for i in range(0, len(raw) - 31, 32): + e = raw[i:i + 32] + if e[0] == 0: + break + attr = e[11] + if e[0] == 0xE5: + lfn = {} + continue + if attr == 0x0F: + part = (e[1:11] + e[14:26] + e[28:32]).decode("utf-16-le", "ignore") + lfn[e[0] & 0x1F] = part.split("\x00")[0].replace("\uffff", "") + continue + if attr & 0x08: + lfn = {} + continue + if lfn: + name = "".join(lfn[k] for k in sorted(lfn)) + else: + base = e[0:8].decode("ascii", "replace").rstrip() + ext = e[8:11].decode("ascii", "replace").rstrip() + name = base + ("." + ext if ext else "") + lfn = {} + if name in (".", ".."): + continue + cluster = (struct.unpack_from(" free - (1 << 30): + raise RuntimeError(f"not enough disk space in {work} for the " + f"{entry['size'] // (1 << 20)} MB ISO") + dest = os.path.join(work, "source.iso") + _copy(f, dest, vol.runs(entry["cluster"], entry["size"]), entry["size"], progress) + return dest, name + raise ValueError("not an ISO, and no FAT partition with an .iso file in it found " + "(extract the ISO with 7z and upload the .iso instead)") + finally: + f.close() + + +def extract_iso(iso, dest): + """Pull the release members out of the ISO with 7z (as the script does).""" + members = list(ISO_MEMBERS.values()) + list(ISO_OPTIONAL.values()) + [ISO_VERSION, ISO_SHA512] + r = subprocess.run(["7z", "x", "-y", "-bd", f"-o{dest}", iso] + members, + capture_output=True, text=True) + if r.returncode not in (0, 1): + raise RuntimeError(f"7z failed ({r.returncode}): {(r.stderr or r.stdout).strip()[-300:]}") + missing = [m for m in ISO_MEMBERS.values() if not os.path.isfile(os.path.join(dest, m))] + if missing: + raise ValueError("not a Blancco Drive Eraser ISO: missing " + ", ".join(missing)) + + +# --- staging (background job) ----------------------------------------------- + +class _Progress: + def __init__(self, source): + self.data = {"state": "running", "source": os.path.basename(source), "step": "", + "percent": None, "message": "", "log": [], "started": uc.now(), + "pid": os.getpid()} + self._last = 0 + self.save() + + def save(self): + uc.write_json(job_file(), self.data) + self._last = time.monotonic() + + def step(self, text): + self.data.update(step=text, percent=None) + self.log(text) + + def log(self, text): + self.data["log"] = (self.data["log"] + [f"{datetime.now():%H:%M:%S} {text}"])[-60:] + self.save() + + def bytes(self, done, total): + if total: + self.data["percent"] = round(100.0 * done / total, 1) + self.data["message"] = f"{done // (1 << 20)} MB" + (f" of {total // (1 << 20)} MB" if total else "") + if time.monotonic() - self._last > 1: + self.save() + + def finish(self, state, message, **extra): + self.data.update(state=state, message=message, finished=uc.now(), percent=None, **extra) + self.data.pop("pid", None) + self.log(message) + + +def job_state(): + data = uc.read_json(job_file(), {}) or {} + if data.get("state") == "running" and not job_alive(data): + data["state"] = "failed" + data["message"] = "the staging job stopped without finishing" + return data + + +def job_alive(data=None): + data = data if data is not None else (uc.read_json(job_file(), {}) or {}) + if data.get("state") != "running": + return False + try: + if uc.job_running(JOB_UNIT): + return True + except OSError: + pass + pid = data.get("pid") + if pid and pid == os.getpid(): + return True + return False + + +def _refuse_while_staging(what): + """The job renames whole folders in staged/ (re-staging a version + replaces its folder), so nothing may read or change them meanwhile.""" + if job_alive(): + raise RuntimeError(f"a staging job is running; {what} when it has finished") + + +def start_stage(path): + if job_alive(): + raise RuntimeError("a staging job is already running") + uc.write_json(job_file(), {"state": "running", "source": os.path.basename(path), + "step": "starting", "log": [], "started": uc.now()}) + try: + uc.start_job(JOB_UNIT, ["services.blancco_update", "stage", path]) + except Exception: + uc.write_json(job_file(), {}) + raise + + +def cancel_stage(): + was_running = (uc.read_json(job_file(), {}) or {}).get("state") == "running" + # systemctl stop waits for the job, which records itself as failed on SIGTERM. + uc.stop_job(JOB_UNIT) + data = uc.read_json(job_file(), {}) or {} + if was_running: + data.update(state="cancelled", message="cancelled", finished=uc.now()) + uc.write_json(job_file(), data) + cleanup_partials() + + +def cleanup_partials(): + """Remove what a stopped job left: staged/.tmp-* and the work folder.""" + root = staged_root() + if os.path.isdir(root): + for name in os.listdir(root): + if name.startswith((".tmp-", ".old-")): + shutil.rmtree(os.path.join(root, name), ignore_errors=True) + shutil.rmtree(os.path.join(_dir(), "work"), ignore_errors=True) + + +def stage(path): + """Background job: vendor ISO / USB image to WEB_ROOT/blancco/staged//.""" + progress = _Progress(path) + root = staged_root() + os.makedirs(root, exist_ok=True) + tmp = os.path.join(root, f".tmp-{_ts()}") + work = os.path.join(_dir(), "work") + shutil.rmtree(work, ignore_errors=True) + os.makedirs(work) + try: + progress.step("Finding the ISO") + iso, iso_name = iso_from_source(path, work, progress.bytes) + progress.log(f"ISO: {iso_name} ({os.path.getsize(iso) // (1 << 20)} MB)") + + progress.step("Extracting release files with 7z") + os.makedirs(tmp) + extract_iso(iso, tmp) + if iso != path: + os.remove(iso) + + version = (_read(os.path.join(tmp, ISO_VERSION)) or b"").decode("ascii", "replace").strip() + cfg = _read(os.path.join(tmp, "arch/boot/config.img")) + vendor_prefs = cpio_member(cfg, "preferences.xml") + if vendor_prefs is None: + raise ValueError("config.img has no preferences.xml") + if not _SAFE_VERSION.match(version): + version = prefs_version(vendor_prefs) or f"unknown-{_ts()}" + progress.data["version"] = version + progress.log(f"Blancco {version}, preferences schema {prefs_version(vendor_prefs)}") + + sha_line = (_read(os.path.join(tmp, ISO_SHA512)) or b"").split() + if sha_line: + progress.step("Checking airootfs.sfs against the ISO's sha512") + got = uc.file_hash(os.path.join(tmp, ISO_MEMBERS["arch/x86_64/airootfs.sfs"]), "sha512") + if got.lower() != sha_line[0].decode("ascii", "replace").lower(): + raise ValueError("airootfs.sfs does not match airootfs.sha512 (corrupt ISO?)") + progress.log("airootfs.sfs sha512 OK") + + progress.step("Laying out the release") + dest = os.path.join(root, f".tmp-{version}-{_ts()}") + os.makedirs(os.path.join(dest, "arch", "x86_64")) + for rel, member in list(ISO_MEMBERS.items()) + list(ISO_OPTIONAL.items()): + src = os.path.join(tmp, member) + if os.path.isfile(src): + os.replace(src, os.path.join(dest, rel)) + shutil.rmtree(tmp, ignore_errors=True) + tmp = dest + + progress.step("Patching the initramfs (keep the network after copytoram)") + initramfs = os.path.join(dest, "initramfs-bde-linux.img") + _atomic_write(initramfs, patch_initramfs(_read(initramfs))) + + # Keep the vendor's config.img + preferences for a later merge. + vdir = _vendor_dir(version) + os.makedirs(vdir, exist_ok=True) + shutil.copyfile(os.path.join(dest, "config.img"), os.path.join(vdir, "config.img")) + _atomic_write(os.path.join(vdir, "preferences.xml"), _vendor_prefs_text(vendor_prefs)) + + meta = {"version": version, "kernel": kernel_version(os.path.join(dest, "vmlinuz-bde-linux")), + "vendor_prefs_version": prefs_version(vendor_prefs), "source": os.path.basename(path), + "iso": os.path.basename(iso_name), "staged": uc.now(), + "rootfs_sha512": sha_line[0].decode() if sha_line else None, + "status": "extracting"} + uc.write_json(os.path.join(dest, "release.json"), meta) + + final = os.path.join(root, version) + if os.path.exists(final): + old = os.path.join(root, f".old-{version}-{_ts()}") + os.replace(final, old) + os.replace(dest, final) + shutil.rmtree(old, ignore_errors=True) + progress.log(f"Replaced the earlier staged {version}") + else: + os.replace(dest, final) + tmp = None + _fix_modes(final) + + progress.step("Writing our preferences into config.img") + try: + apply_prefs(version) + except StageBlocked as e: + progress.finish("blocked", str(e), version=version) + return 2 + progress.finish("done", f"Blancco {version} staged and ready to test", version=version) + return 0 + except (Exception, SystemExit) as e: + msg = str(e) if not isinstance(e, SystemExit) else "stopped" + progress.finish("failed", f"Staging failed: {msg}") + return 1 + finally: + if tmp: + shutil.rmtree(tmp, ignore_errors=True) + shutil.rmtree(work, ignore_errors=True) + + +def _vendor_prefs_text(vendor): + """The vendor preferences without the NUL padding and filler comment, + as a file a human can merge against.""" + end = vendor.find(b"") + return (vendor[:end + 7] if end >= 0 else vendor.replace(b"\x00", b"")).rstrip() + b"\n" + + +def vendor_prefs_file(version): + path = os.path.join(_vendor_dir(_check_version(version)), "preferences.xml") + return path if os.path.isfile(path) else None + + +def apply_prefs(version): + """Write our preferences into a staged release: config.img (vendor + copy + ours), preferences.xml, config-clean.xml. Our preferences are the + merged file uploaded for this version, else the ones served now. + Raises StageBlocked when they fail the version guard.""" + _check_version(version) + folder = os.path.join(staged_root(), version) + meta_path = os.path.join(folder, "release.json") + meta = uc.read_json(meta_path, {}) or {} + vendor_cfg = _read(os.path.join(_vendor_dir(version), "config.img")) + if vendor_cfg is None: + raise ValueError(f"no vendor config.img kept for {version}; stage it again") + merged = merged_prefs_path(version) + src = merged if os.path.isfile(merged) else active_prefs_path() + ours = _read(src) + if not ours: + raise ValueError(f"no preferences at {src}") + try: + cfg = patch_config_img(vendor_cfg, ours) + except StageBlocked as e: + meta.update(status="blocked", blocked=str(e), prefs_source=src) + uc.write_json(meta_path, meta) + raise + except ValueError as e: + # Not a merge problem (no report share, too long for the vendor + # file): the row says why instead of sitting at "extracting". + meta.update(status="failed", error=f"our preferences ({src}): {e}", prefs_source=src) + uc.write_json(meta_path, meta) + raise + _atomic_write(os.path.join(folder, "config.img"), cfg) + _atomic_write(os.path.join(folder, "preferences.xml"), ours) + _atomic_write(os.path.join(folder, "config-clean.xml"), config_clean(cfg)) + meta.pop("blocked", None) + meta.pop("error", None) + meta.update(status="ready", prefs_version=prefs_version(ours), prefs_source=src, + ready=uc.now()) + uc.write_json(meta_path, meta) + _fix_modes(folder) + return meta + + +def save_merged_prefs(version, data): + """A human-merged preferences.xml for a staged version: check it, keep it, + and finish staging with it.""" + _check_version(version) + _refuse_while_staging("upload the merged preferences") + if not os.path.isdir(os.path.join(staged_root(), version)): + raise ValueError(f"{version} is not staged") + try: + ET.fromstring(data.replace(b"\x00", b"")) + except ET.ParseError as e: + raise ValueError(f"not well-formed XML: {e}") + vendor = _read(os.path.join(_vendor_dir(version), "preferences.xml")) or b"" + if _guard_version(data) != _guard_version(vendor): + raise ValueError(f"the merged file says {prefs_version(data) or 'no version'}, the " + f"release's preferences are {prefs_version(vendor)}: merge onto " + "the new vendor file, including its block") + vendor_cfg = _read(os.path.join(_vendor_dir(version), "config.img")) + if vendor_cfg is None: + raise ValueError(f"no vendor config.img kept for {version}; stage it again") + # Every other check (report share, fits the vendor file) before the file + # is kept: a stored merged file wins over the live preferences from then + # on, also for a later re-stage of this version. + patch_config_img(vendor_cfg, data) + with _lock: + _atomic_write(merged_prefs_path(version), data) + return apply_prefs(version) + + +def delete_staged(version): + _check_version(version) + _refuse_while_staging("delete it") + test = test_entry() + if test and test.get("version") == version: + raise RuntimeError(f"the test menu entry boots {version}; remove it first") + path = os.path.join(staged_root(), version) + if not os.path.isdir(path): + raise ValueError(f"{version} is not staged") + shutil.rmtree(path) + shutil.rmtree(_vendor_dir(version), ignore_errors=True) + + +# --- file helpers ----------------------------------------------------------- + +def _atomic_write(path, data): + tmp = f"{path}.tmp-{os.getpid()}" + with open(tmp, "wb") as f: + f.write(data) + os.chmod(tmp, 0o644) + os.replace(tmp, path) + + +def _fix_modes(folder): + """Apache must read everything: dirs 755, files 644.""" + for dirpath, dirnames, filenames in os.walk(folder): + os.chmod(dirpath, 0o755) + for name in filenames: + os.chmod(os.path.join(dirpath, name), 0o644) + + +def _link_or_copy(src, dst): + """Hard link src at dst (same filesystem: instant, no space), else copy.""" + os.makedirs(os.path.dirname(dst), exist_ok=True) + try: + os.link(src, dst) + except OSError: + shutil.copy2(src, dst) + + +def _place(src, dst): + """Put src's content at dst atomically: link to a temp name, rename.""" + tmp = os.path.join(os.path.dirname(dst), f".{os.path.basename(dst)}.new-{os.getpid()}") + if os.path.lexists(tmp): + os.remove(tmp) + _link_or_copy(src, tmp) + os.replace(tmp, dst) + + +def _backup_live(reason, files): + """Hard-link the live copies of files into backup--/.""" + live = current() + label = re.sub(r"[^0-9A-Za-z._-]", "_", str(live.get("version") or "unknown")) + base = name = f"backup-{label}-{_ts()}" + n = 1 + while os.path.exists(os.path.join(blancco_root(), name)): + n += 1 + name = f"{base}-{n}" + folder = os.path.join(blancco_root(), name) + os.makedirs(folder) + saved = [] + for rel in files: + src = os.path.join(blancco_root(), rel) + if os.path.isfile(src): + _link_or_copy(src, os.path.join(folder, rel)) + saved.append(rel) + meta = {k: live.get(k) for k in ("version", "kernel", "prefs_version", "source", "staged", + "activated", "inferred") if live.get(k) is not None} + meta.update(backed_up=uc.now(), reason=reason, files_saved=saved) + uc.write_json(os.path.join(folder, "release.json"), meta) + _fix_modes(folder) + return name + + +def _swap_in(folder, files): + """Place each file from folder into the live release. The callers undo a + failure by swapping the backup they just took back in.""" + for rel in files: + _place(_find_release_file(folder, rel), os.path.join(blancco_root(), rel)) + + +def _write_live_meta(meta): + meta["fingerprint"] = _fingerprint(blancco_root()) + path = os.path.join(blancco_root(), "release.json") + uc.write_json(path, meta) + os.chmod(path, 0o644) + + +def activate(version): + """Make a staged release live. Returns (backup name, files swapped, + test entry note): the note is True when the release's test entry was + removed from the menu, or the error text when that edit failed (the + release is live either way).""" + _check_version(version) + _refuse_while_staging("activate") + folder = os.path.join(staged_root(), version) + with _lock: + meta = uc.read_json(os.path.join(folder, "release.json"), {}) or {} + if meta.get("status") != "ready": + raise RuntimeError(f"{version} is not ready to activate ({meta.get('status') or 'unknown'})") + info = release_info(folder) + if info["missing"]: + raise RuntimeError(f"{version} is missing " + ", ".join(info["missing"])) + files = list(RELEASE_FILES) + [r for r in OPTIONAL_FILES if r in info["files"]] + backup = _backup_live(f"before activating {version}", files) + try: + _swap_in(folder, files) + except Exception as e: + _swap_in(os.path.join(blancco_root(), backup), + [r for r in files if _find_release_file(os.path.join(blancco_root(), backup), r)]) + raise RuntimeError(f"activation failed and was undone: {e}") + meta.update(activated=uc.now(), backup=backup) + _write_live_meta(meta) + # The staged copies are hard links of what is live now; the folder + # goes so a later edit there can never reach the live files. + shutil.rmtree(folder, ignore_errors=True) + removed_entry = False + test = test_entry() + if test and test.get("version") == version: + try: + _remove_test_entry_locked() + removed_entry = True + except (OSError, RuntimeError) as e: + removed_entry = f"the test entry is still in the menu ({e}); remove it" + return backup, files, removed_entry + + +def rollback(name): + """Put a backup's files back live (after backing up what is live now).""" + if not re.match(r"^backup-[0-9A-Za-z._-]+$", name or ""): + raise ValueError("bad backup name") + folder = os.path.join(blancco_root(), name) + with _lock: + if not os.path.isdir(folder): + raise ValueError("no such backup") + info = release_info(folder) + if info["missing"]: + raise RuntimeError(f"{name} is incomplete, missing " + ", ".join(info["missing"])) + files = list(RELEASE_FILES) + [r for r in OPTIONAL_FILES if r in info["files"]] + backup = _backup_live(f"before rolling back to {name}", files) + try: + _swap_in(folder, files) + except Exception as e: + _swap_in(os.path.join(blancco_root(), backup), + [r for r in files if _find_release_file(os.path.join(blancco_root(), backup), r)]) + raise RuntimeError(f"rollback failed and was undone: {e}") + meta = {k: v for k, v in info.items() if k in ("version", "kernel", "prefs_version", + "source", "staged", "inferred")} + meta.update(restored_from=name, activated=uc.now(), backup=backup) + _write_live_meta(meta) + try: + _release_backup(folder, files) + except OSError: + pass # the rollback itself is done; a backup left whole is harmless + return backup, files + + +def _release_backup(folder, files): + """After a rollback the backup's files ARE the live files (hard links). + Drop the backup's names for them, so an in-place write to a live file + (a playbook run's cp -r boot-tools/blancco/*, or a cp by hand) cannot + also change a backup. Nothing is lost: what is live is backed up again + the next time something replaces it.""" + for rel in files: + path = _find_release_file(folder, rel) + if path: + os.remove(path) + meta_path = os.path.join(folder, "release.json") + left = [os.path.join(d, n) for d, _, ns in os.walk(folder) for n in ns] + if all(p == meta_path for p in left): + shutil.rmtree(folder, ignore_errors=True) + else: + # A hand-made backup with extra files (preferences.save, ...): keep + # those, say what happened to the rest. + uc.write_json(meta_path, {"reason": f"restored to live {uc.now()}; only the files " + "that are not part of a release remain"}) + + +def delete_backup(name): + if not re.match(r"^backup-[0-9A-Za-z._-]+$", name or ""): + raise ValueError("bad backup name") + folder = os.path.join(blancco_root(), name) + if not os.path.isdir(folder): + raise ValueError("no such backup") + shutil.rmtree(folder) + + +# --- test entry in the iPXE menu -------------------------------------------- + +def _read_menu(): + path = menu_path() + # surrogateescape: any byte a hand edit left in the menu round-trips. + with open(path, newline="", encoding="utf-8", errors="surrogateescape") as f: + text = f.read() + nl = "\r\n" if "\r\n" in text else "\n" + return path, text, nl + + +def _write_menu(path, text, tag): + """Back up the menu next to itself, then replace it atomically with the + same owner and mode.""" + st = os.stat(path) + backup = f"{path}.bak-{_ts()}-{tag}" + shutil.copy2(path, backup) + tmp = f"{path}.tmp-{os.getpid()}" + try: + with open(tmp, "w", newline="", encoding="utf-8", errors="surrogateescape") as f: + f.write(text) + os.chmod(tmp, st.st_mode & 0o7777) + try: + os.chown(tmp, st.st_uid, st.st_gid) + except PermissionError: + pass + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.remove(tmp) + return backup + + +def _block(lines, label): + """The kernel line and the initrd lines right after it, in a menu label's + section (up to the next label).""" + try: + i = lines.index(f":{label}") + except ValueError: + return None + kernel, initrds = None, [] + for line in lines[i + 1:]: + if line.startswith(":"): + break + if kernel is None: + if line.startswith("kernel "): + kernel = line + continue + if line.startswith("initrd "): + initrds.append(line) + else: + break + return (kernel, initrds) if kernel else None + + +def _retarget(line, version, goto): + """A menu line pointed at the staged release files, failing to goto.""" + base = f"blancco/staged/{version}/" + names = "|".join(re.escape(n) for n in _MENU_RELEASE_NAMES) + line = re.sub(r"(http://[^/\s]+/)blancco/(?=(?:%s)(?:\s|$))" % names, r"\g<1>" + base, line) + line = re.sub(r"(archiso_http_srv=http://[^/\s]+/)blancco/", r"\g<1>" + base, line) + line = re.sub(r"\s*\|\|\s*goto\s+\S+\s*$", "", line) + return f"{line} || goto {goto}" + + +def test_entry(): + """{version, item, efi} of the staged test entry in the menu, or None. + efi is False when the entry was added with its UEFI branch off.""" + try: + _, text, _ = _read_menu() + except OSError: + return None + m = re.search(r"^item\s+%s\s+.*$" % TEST_LABEL, text, re.M) + if not m and TEST_BEGIN not in text: + return None + v = re.search(r"blancco\.base=/blancco/staged/([0-9A-Za-z._-]+)", text) or \ + re.search(r"/blancco/staged/([0-9A-Za-z._-]+)/", text) + return {"version": v.group(1) if v else None, "item": m.group(0).strip() if m else None, + "efi": _EFI_OFF not in text} + + +_EFI_OFF = "echo UEFI test boot is off:" + + +def build_test_entry(text, version, efi_ok=True): + """(menu text with the test entry added, the entry's lines). efi_ok + False: the UEFI initrd's init does not read blancco.base=, so the UEFI + branch would boot the LIVE release while the screen says STAGED; it + says so and goes back to the menu instead.""" + nl = "\r\n" if "\r\n" in text else "\n" + lines = text.split(nl) + efi = _block(lines, "blancco_efi") + bios = _block(lines, "blancco_bios") + if not efi: + raise RuntimeError("the menu has no :blancco_efi kernel line to copy") + items = [i for i, l in enumerate(lines) if re.match(r"^item\s+blancco\s", l)] + if len(items) != 1: + raise RuntimeError("cannot find the menu's 'item blancco' line") + fail = f"{TEST_LABEL}_failed" + kernel = _retarget(efi[0], version, fail) + kernel = kernel.replace(f" || goto {fail}", f" blancco.base=/blancco/staged/{version} || goto {fail}") + block = [ + TEST_BEGIN, + f":{TEST_LABEL}", + f"echo Loading STAGED Blancco {version} for testing (not the active release)...", + f"iseq ${{platform}} pcbios && goto {TEST_LABEL}_bios ||", + ] + if efi_ok: + block += [kernel] + [_retarget(l, version, fail) for l in efi[1]] + [f"boot || goto {fail}"] + else: + block += [ + f"{_EFI_OFF} the UEFI initrd predates blancco.base= and would boot the LIVE release.", + "echo Rebuild the initrd from playbook/blancco-init.sh and add the test entry again,", + "echo or test this release on a legacy BIOS boot.", + "prompt --timeout 30000 Press any key to return to the menu... ||", + "goto menu", + ] + block += [f":{TEST_LABEL}_bios"] + if bios: + block += [_retarget(bios[0], version, fail)] + [_retarget(l, version, fail) for l in bios[1]] + block += [f"boot || goto {fail}"] + else: + block += ["echo No legacy BIOS Blancco entry in this menu to copy."] + block += [ + f":{fail}", + "echo", + f"echo Staged Blancco {version} boot FAILED: ${{errno}}", + "prompt --timeout 30000 Press any key to return to the menu... ||", + "goto menu", + TEST_END, + ] + item = f"item {TEST_LABEL:<20}Blancco Drive Eraser {version} (STAGED - TEST)" + lines.insert(items[0] + 1, item) + # At the end of the script, where nothing falls into it (a guard jump + # inside the markers when the script's last line would fall through). + while lines and not lines[-1].strip(): + lines.pop() + last = lines[-1].strip() if lines else "" + if not re.match(r"^(exit|goto\s|reboot|shell|poweroff)", last): + block.insert(1, "goto menu") + lines += [""] + block + [""] + return nl.join(lines), block + + +def add_test_entry(version): + _check_version(version) + folder = os.path.join(staged_root(), version) + meta = uc.read_json(os.path.join(folder, "release.json"), {}) or {} + _refuse_while_staging("add the test entry") + if meta.get("status") != "ready": + raise RuntimeError(f"{version} is not ready to test ({meta.get('status') or 'not staged'})") + efi_ok = initrd_supports_base()["state"] != "no" + with _lock: + path, text, _ = _read_menu() + if test_entry(): + text = _strip_test_entry(text) + new, _ = build_test_entry(text, version, efi_ok) + return _write_menu(path, new, "pre-blancco-staged") + + +def _strip_test_entry(text): + nl = "\r\n" if "\r\n" in text else "\n" + lines = text.split(nl) + out, skip = [], False + for line in lines: + if line == TEST_BEGIN: + skip = True + # The blank line added before the block. + if out and not out[-1].strip(): + out.pop() + continue + if skip: + if line == TEST_END: + skip = False + continue + if re.match(r"^item\s+%s\s" % TEST_LABEL, line): + continue + out.append(line) + if skip: + raise RuntimeError("test entry has no end marker; edit the menu by hand") + return nl.join(out) + + +def _remove_test_entry_locked(): + path, text, _ = _read_menu() + new = _strip_test_entry(text) + if new == text: + return None + return _write_menu(path, new, "pre-blancco-staged-remove") + + +def remove_test_entry(): + with _lock: + return _remove_test_entry_locked() + + +def menu_efi_initrd(): + """File name of the UEFI path's init (kexec-initrd*.img) in the menu.""" + try: + _, text, nl = _read_menu() + except OSError: + return None + block = _block(text.split(nl), "blancco_efi") + for line in (block[1] if block else []): + m = re.search(r"/blancco/(kexec-initrd[^\s/]*\.img)", line) + if m: + return m.group(1) + return None + + +def initrd_supports_base(): + """Whether the UEFI initrd's init knows blancco.base= (else the UEFI test + boot gets the ACTIVE release). Cached by file size + mtime.""" + name = menu_efi_initrd() + if not name: + return {"state": "unknown", "file": None, "detail": "no :blancco_efi initrd in the menu"} + path = os.path.join(blancco_root(), name) + try: + st = os.stat(path) + except OSError: + return {"state": "unknown", "file": name, "detail": "file not found"} + cache_path = os.path.join(_dir(), "initrd-check.json") + key = f"{name}:{st.st_size}:{int(st.st_mtime)}" + cache = uc.read_json(cache_path, {}) or {} + if cache.get("key") == key: + return cache["result"] + result = {"state": "unknown", "file": name, "detail": "not gzip"} + try: + with open(path, "rb") as f: + magic = f.read(2) + if magic == b"\x1f\x8b": + found, tail = False, b"" + with gzip.open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + if b"blancco.base=" in tail + block: + found = True + break + tail = block[-32:] + result = {"state": "yes" if found else "no", "file": name, + "detail": "init reads blancco.base=" if found else + "init predates blancco.base= (rebuild the initrd from playbook/blancco-init.sh)"} + except (OSError, EOFError, gzip.BadGzipFile) as e: + result = {"state": "unknown", "file": name, "detail": str(e)} + uc.write_json(cache_path, {"key": key, "result": result}) + return result + + +# --- entry point for the background job ------------------------------------- + +def _on_term(signum, frame): + raise SystemExit(143) + + +def main(argv): + if len(argv) == 2 and argv[0] == "stage": + signal.signal(signal.SIGTERM, _on_term) + return stage(argv[1]) + print("usage: python3 -m services.blancco_update stage FILE", file=sys.stderr) + return 64 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/webapp/services/bprt_sync.py b/webapp/services/bprt_sync.py new file mode 100644 index 0000000..9f2036e --- /dev/null +++ b/webapp/services/bprt_sync.py @@ -0,0 +1,1467 @@ +"""Bring the vendor BPRT folder from the work PC into _shared/Applications/BPRT. + +BPRT (every image's Deploy/Applications/BPRT links to it; PESetup copies it to +every PC while it images) is a vendor folder of about 7.7 GB: the six MCL +packages, the Office zips and the bulk-enroll orchestrator scripts. The vendor +keeps it current on the user's GE work PC with MediaCreator Lite; this server +is only reachable from there over the GE network (this webapp and SSH, no +SMB), and never reaches out itself. So the user runs +scripts/windows/Sync-BPRT.ps1 there, which: + + 1. GETs manifest.json (path, size, sha256 of every vendor file here); + 2. hashes its own copy and uploads only new or changed files, plus a + sync.json listing the complete intended vendor folder, with one sftp + batch into BPRT-incoming//. + +This module then, each step a background job (python3 -m services.bprt_sync): + + * manifest hashes the live folder (cached by path, size and mtime in + UPDATES_DIR/bprt/hashcache.json; only the first run reads all + 7.7 GB); + * validate checks an incoming batch: every listed file uploaded with the + right hash or unchanged here, the vendor's one-package-per-prefix + rule (services/mcl_ppkg.py), the scripts the orchestrator calls; + and works out the diff; + * activate builds BPRT.new from hard links (live files that did not change, + plus the incoming ones), applies the site overlay, and swaps it + in with two renames: BPRT to BPRT-archive/full-, BPRT.new to + BPRT. Rollback swaps an archive back the same way. + * unzip the fallback when SSH is not available: a zip uploaded on the + page is unpacked into a new incoming batch, then validated. + +Hard links: an archive shares every unchanged file with the live folder, so +it costs only the files that changed. Nothing here edits a file in place; +replace files, never rewrite them, or the archives change with them. +""" +from __future__ import annotations + +import errno +import fcntl +import hashlib +import json +import os +import re +import shutil +import signal +import stat +import sys +import time +import zipfile +from collections import Counter +from datetime import datetime, timezone + +from services import mcl_ppkg +from services import updates_common as uc + +UNIT = "pxe-bprt-sync" + +# The site overlay: files in the live BPRT that are ours, not the vendor's. +# manifest.json leaves them out (the work PC never sees or replaces them) and +# activation carries them into every new BPRT from the live one. +# Select-PCConfig.ps1 our shim, repo playbook/shopfloor-setup/BPRT/ +# Select-PCConfig.ps1: returns the tenant and +# purpose startnet.cmd pre-seeded, else runs the +# vendor dialog, which is kept beside it as +# Select-PCConfig-vendor.ps1 (VENDOR_RENAMED); +# Register-BulkEnrollTask-Standard.ps1 registers the orchestrator; specialize Order 39 +# of playbook/FlatUnattendW10.xml and +# FlatUnattendW10-engineer.xml runs it. It exists +# only on the share (no repo copy); +# Register-BulkEnrollTask.ps1 the shopfloor -ManualFallback variant. Nothing +# calls it since the shopfloor unattend dropped +# its Order 16 (2026-08-06); kept as it was. +OVERLAY = ("Select-PCConfig.ps1", "Register-BulkEnrollTask-Standard.ps1", + "Register-BulkEnrollTask.ps1") +# Vendor files the overlay displaces, and the name the vendor's copy keeps. +VENDOR_RENAMED = {"Select-PCConfig.ps1": "Select-PCConfig-vendor.ps1"} +# Our shim calls the vendor copy by this name; the vendor dialog never does. +SHIM_MARKER = b"Select-PCConfig-vendor.ps1" +# Overlay files enrollment cannot work without (the unattends call it). +OVERLAY_REQUIRED = ("Register-BulkEnrollTask-Standard.ps1",) + +# Vendor files the bulk-enroll flow calls by path (C:\Deploy\Applications\BPRT\...): +# a drop without one of them breaks enrollment on every PC. Read from +# Start-BulkEnrollOrchestrator.ps1 and the scripts it starts, plus the two the +# unattends reach. Translations\*.psd1 are optional (WCD-Helpers.ps1 falls +# back to English). +REQUIRED = ( + "Start-BulkEnrollOrchestrator.cmd", # what Register-BulkEnrollTask-Standard.ps1 registers + "Start-BulkEnrollOrchestrator.ps1", + "WCD-Helpers.ps1", # dot-sourced by the orchestrator and every dialog + "Show-ImageUntilNetworkConnected.ps1", + "Waiting For Internet Connection.png", + "Select-PCConfig.ps1", # the vendor dialog, kept as Select-PCConfig-vendor.ps1 + "GE Aerospace Transparent Logo.png", + "Show-PleaseWait.ps1", + "Translations.json", + "ForegroundWindowWithProgress.exe", + "Insert-BPRTToPPKG.ps1", + "Send-OEMSkipPA.ps1", + "Wait-IntuneEnrollmentAndReboot.ps1", + "Show-VanillaConfirmation.ps1", # the unattends' 'vanilla' Shift+F10 alias +) + +# Windows PowerShell 5.1 reads a script without a BOM as ANSI, which garbles +# any non-ASCII text in it; the vendor ships five such scripts. Activation +# gives those a UTF-8 BOM (what was done by hand for the v2.19 drop), and the +# manifest also reports their hash without it so the work PC does not see +# them as changed. +BOM = b"\xef\xbb\xbf" +SCRIPT_EXTS = (".ps1", ".psm1", ".psd1") + +_BATCH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$") +_ARCHIVE = re.compile(r"^full-\d{8}-\d{6}(-\d+)?$") +_WIN_BAD = re.compile(r'[<>:"|?*\\\x00-\x1f]') +_DEFAULT_FILE_MODE = 0o744 # what the share's BPRT files carry (see mcl_ppkg) + + +# --- paths ------------------------------------------------------------------- + +def bprt_dir(): + return mcl_ppkg.bprt_dir() + + +def new_dir(): + return bprt_dir() + ".new" + + +def archive_root(): + return mcl_ppkg.archive_dir() + + +def incoming_root(): + return os.path.join(os.path.dirname(bprt_dir()), "BPRT-incoming") + + +def uploads_dir(): + return os.path.join(incoming_root(), ".uploads") + + +def _state(*parts): + return uc.state_dir("bprt", *parts) + + +def _path(name): + return os.path.join(_state(), name) + + +def _stamp(): + return datetime.now().strftime("%Y%m%d-%H%M%S") + + +def check_batch(name): + name = str(name or "") + if not _BATCH.match(name) or name.endswith(".part"): + raise ValueError(f"{name or '(blank)'} is not an incoming batch name") + return name + + +def batch_dir(name): + return os.path.join(incoming_root(), check_batch(name)) + + +def _owner(): + """(uid, gid, mode) of the live BPRT folder, else of its parent.""" + for p in (bprt_dir(), os.path.dirname(bprt_dir())): + try: + st = os.stat(p) + return st.st_uid, st.st_gid, stat.S_IMODE(st.st_mode) + except OSError: + continue + return None, None, 0o755 + + +def _match_owner(path, mode): + uid, gid, _ = _owner() + try: + if uid is not None: + os.chown(path, uid, gid) + os.chmod(path, mode) + except OSError: + pass # not root (local tests): the files are ours already + + +def ensure_incoming(): + """BPRT-incoming, writable by the share owner (the SSH user uploads as pxe).""" + path = incoming_root() + if not os.path.isdir(path): + os.makedirs(path, exist_ok=True) + _match_owner(path, _owner()[2] or 0o755) + return path + + +# --- walking and hashing ----------------------------------------------------- + +def walk(root): + """({relpath: stat} of the regular files under root, [skipped relpaths]). + Relpaths use forward slashes; symlinks and special files are skipped.""" + files, skipped = {}, [] + if not os.path.isdir(root): + return files, skipped + for here, dirs, names in os.walk(root): + dirs.sort() + rel_here = os.path.relpath(here, root) + for d in list(dirs): + if os.path.islink(os.path.join(here, d)): + dirs.remove(d) + skipped.append(_rel(rel_here, d) + "/") + for n in sorted(names): + rel = _rel(rel_here, n) + st = os.lstat(os.path.join(here, n)) + if stat.S_ISREG(st.st_mode): + files[rel] = st + else: + skipped.append(rel) + return files, skipped + + +def _rel(rel_here, name): + return name if rel_here == "." else f"{rel_here.replace(os.sep, '/')}/{name}" + + +def _is_script(rel): + return rel.lower().endswith(SCRIPT_EXTS) + + +def hash_file(path, rel=None, progress=None): + """(sha256, sha256 of the content after a UTF-8 BOM or None), uppercase, + in one read. The second is only computed for scripts that start with a + BOM.""" + h, h2 = hashlib.sha256(), None + script = _is_script(rel or path) + with open(path, "rb") as f: + first = True + for block in iter(lambda: f.read(1 << 22), b""): + h.update(block) + if first: + first = False + if script and block.startswith(BOM): + h2 = hashlib.sha256(block[len(BOM):]) + elif h2 is not None: + h2.update(block) + if progress: + progress(len(block)) + return h.hexdigest().upper(), (h2.hexdigest().upper() if h2 else None) + + +def _key(rel, st): + return f"{rel}|{st.st_size}|{st.st_mtime_ns}" + + +class HashCache: + """sha256 per (relpath, size, mtime), shared by the live folder, the + archives and BPRT.new: a hard link keeps size and mtime, so a file that + only moved between them is never read again.""" + + def __init__(self): + data = uc.read_json(_path("hashcache.json"), {}) or {} + self.entries = data.get("entries") if isinstance(data.get("entries"), dict) else {} + self.dirty = False + + def get(self, rel, st): + e = self.entries.get(_key(rel, st)) + if e and time.time() - e.get("seen", 0) > 86400: + e["seen"] = int(time.time()) # still in use: keep it past the pruning age + self.dirty = True + return e + + def put(self, rel, st, sha, nobom): + self.entries[_key(rel, st)] = {"sha256": sha, "nobom": nobom, "seen": int(time.time())} + self.dirty = True + + def hash(self, root, rel, st, progress=None): + """Cached entry for root/rel, hashing it on a miss. A file that + changed while it was read is returned but not cached.""" + e = self.get(rel, st) + if e: + return e + path = os.path.join(root, *rel.split("/")) + sha, nobom = hash_file(path, rel, progress) + after = os.lstat(path) + e = {"sha256": sha, "nobom": nobom} + if (after.st_size, after.st_mtime_ns) == (st.st_size, st.st_mtime_ns): + self.put(rel, st, sha, nobom) + return e + + def save(self, max_age_days=180): + if not self.dirty: + return + cutoff = time.time() - max_age_days * 86400 + self.entries = {k: v for k, v in self.entries.items() if v.get("seen", 0) >= cutoff} + uc.write_json(_path("hashcache.json"), {"entries": self.entries}) + self.dirty = False + + +def _same(entry, sha): + """True when a file with this cache entry is the vendor file with this + sha256 (with or without the BOM we add).""" + sha = str(sha or "").upper() + return bool(sha) and sha in (entry.get("sha256"), entry.get("nobom")) + + +# --- the live folder --------------------------------------------------------- + +def _lc(names): + return {n.lower() for n in names} + + +def is_shim(path): + try: + with open(path, "rb") as f: + return SHIM_MARKER.lower() in f.read(1 << 20).lower() + except OSError: + return False + + +def vendor_view(root, files): + """{vendor path: relpath in root} for a BPRT folder: the overlay left + out, Select-PCConfig-vendor.ps1 under the vendor's own name. A folder the + overlay was never applied to still has the vendor's Select-PCConfig.ps1.""" + overlay = _lc(OVERLAY) + back = {v.lower(): k for k, v in VENDOR_RENAMED.items()} + out = {} + for rel in files: + low = rel.lower() + if low in back: + out[back[low]] = rel + elif low not in overlay: + out[rel] = rel + for vendor, kept in VENDOR_RENAMED.items(): + if vendor not in out and vendor in files and not is_shim(os.path.join(root, vendor)): + out[vendor] = vendor + return out + + +def overlay_sources(root, files): + """{overlay name: absolute source path} found in a BPRT folder; our shim + falls back to the repo copy when the webapp runs from a checkout.""" + out = {} + for name in OVERLAY: + path = os.path.join(root, name) + if name in files and (name not in VENDOR_RENAMED or is_shim(path)): + out[name] = path + if "Select-PCConfig.ps1" not in out: + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + repo = os.path.join(here, "..", "playbook", "shopfloor-setup", "BPRT", "Select-PCConfig.ps1") + if is_shim(repo): + out["Select-PCConfig.ps1"] = os.path.abspath(repo) + return out + + +def manifest(build=False, progress=None): + """What manifest.json serves: every vendor file of the live BPRT with size + and sha256 (and sha256_nobom where we added a BOM). Returns None when a + file is not hashed yet and build is False (the caller starts the job).""" + root = bprt_dir() + if os.path.islink(root) or not os.path.isdir(root): + raise RuntimeError(f"{root} is missing or not a folder") + files, _ = walk(root) + view = vendor_view(root, files) + cache = HashCache() + missing = [rel for rel in view.values() if not cache.get(rel, files[rel])] + if missing and not build: + return None + if progress: + progress.start(sum(files[r].st_size for r in missing), len(missing)) + out = [] + for vpath in sorted(view, key=str.lower): + rel = view[vpath] + st = files[rel] + if progress: + progress.current(vpath) + e = cache.hash(root, rel, st, progress.add if progress else None) + row = {"path": vpath, "size": st.st_size, "sha256": e["sha256"]} + if e.get("nobom"): + row["sha256_nobom"] = e["nobom"] + if rel != vpath: + row["live_name"] = rel + out.append(row) + cache.save() + return { + "format": 1, + "generated": uc.now(), + "bprt": root, + "incoming": incoming_root(), + # Never uploaded: the site files the vendor does not also ship. + "overlay": [n for n in OVERLAY if n not in VENDOR_RENAMED], + "renamed": dict(VENDOR_RENAMED), + "count": len(out), + "bytes": sum(r["size"] for r in out), + "files": out, + } + + +def _bom_less_non_ascii(path): + """True for a script PowerShell 5.1 would misread: valid UTF-8 with + non-ASCII text and no BOM.""" + try: + with open(path, "rb") as f: + data = f.read() + except OSError: + return False + if data.startswith(BOM) or data.isascii(): + return False + try: + data.decode("utf-8") + except UnicodeDecodeError: + return False # ANSI already: PowerShell reads it as written + return True + + +def live_summary(): + """Facts for the page about the live BPRT (no hashing).""" + root = bprt_dir() + out = {"path": root, "exists": os.path.isdir(root), "is_link": os.path.islink(root), + "count": 0, "bytes": 0, "newest": None, "overlay": [], "missing_required": [], + "bom_needed": [], "skipped": []} + if not out["exists"]: + return out + files, out["skipped"] = walk(root) + view = vendor_view(root, files) + out["count"] = len(files) + out["bytes"] = sum(st.st_size for st in files.values()) + out["newest"] = max((st.st_mtime for st in files.values()), default=None) + srcs = overlay_sources(root, files) + for name in OVERLAY: + row = {"name": name, "present": name in files, "ok": name in srcs and srcs[name].startswith(root)} + if name in VENDOR_RENAMED: + row["vendor_copy"] = VENDOR_RENAMED[name] in files + row["is_shim"] = row["ok"] + out["overlay"].append(row) + out["missing_required"] = [r for r in REQUIRED if r not in view] + out["bom_needed"] = [rel for rel in files if _is_script(rel) + and _bom_less_non_ascii(os.path.join(root, *rel.split("/")))] + return out + + +# --- job state --------------------------------------------------------------- + +def job_state(): + state = uc.read_json(_path("job.json"), {}) or {} + running = bool(state.get("state") == "running" and uc.job_running(UNIT)) + if state.get("state") == "running" and not running: + state["state"] = "interrupted" + state["running"] = running + total = state.get("total_bytes") or 0 + done = state.get("done_bytes") or 0 + state["percent"] = min(100.0, round(100.0 * done / total, 1)) if total else None + return state + + +def _job_update(**kw): + state = uc.read_json(_path("job.json"), {}) or {} + state.update(kw) + uc.write_json(_path("job.json"), state) + + +class Progress: + """Byte progress of a job, written to job.json at most once a second.""" + + def __init__(self, step=""): + self.done = 0 + self.last = 0.0 + if step: + _job_update(step=step) + + def start(self, total, count=0): + self.done = 0 + _job_update(total_bytes=total, done_bytes=0, files=count) + + def current(self, name): + self.name = name + + def add(self, n): + self.done += n + if time.time() - self.last >= 1: + self.last = time.time() + _job_update(done_bytes=self.done, current=getattr(self, "name", "")) + + def step(self, text): + _job_update(step=text, current="") + + +def busy(): + """Kind of the running job, or ''.""" + st = job_state() + return st.get("kind", "") if st.get("running") else "" + + +def start(kind, arg=""): + """Start a background job; one at a time (they all read or swap BPRT).""" + if kind not in ("manifest", "validate", "activate", "unzip"): + raise ValueError(f"unknown job {kind}") + if busy() or uc.job_running(UNIT): + raise RuntimeError(f"a BPRT job ({busy() or 'unknown'}) is still running") + if kind in ("validate", "activate"): + check_batch(arg) + if kind == "unzip": + clean_zip_name(arg) # also keeps a name like --help out of the job's argv + if kind == "activate": + res = validation(arg) + if not res or res.get("state") != "valid": + raise RuntimeError("validate this batch first; only a valid batch can be activated") + uc.write_json(_path("job.json"), {"kind": kind, "arg": arg, "state": "running", + "started": uc.now(), "step": "Starting"}) + try: + uc.start_job(UNIT, ["services.bprt_sync", kind] + ([arg] if arg else [])) + except Exception as ex: + if not uc.job_running(UNIT): # lost a race to another start: leave its state alone + _job_update(state="error", finished=uc.now(), error=str(ex)[:400]) + raise + + +def run_job(kind, arg=""): + """Body of a job (also called directly by tests): runs it and records the + outcome in job.json.""" + _job_update(kind=kind, arg=arg, state="running", error="", result="", + total_bytes=0, done_bytes=0, current="") + try: + if kind == "manifest": + m = manifest(build=True, progress=Progress("Hashing the live BPRT")) + result = f"{m['count']} files hashed" + elif kind == "validate": + res = validate(arg, Progress("Validating")) + result = res["state"] + elif kind == "activate": + res = activate(arg, Progress("Building BPRT.new")) + result = f"activated, previous BPRT in {res['archive']}" + elif kind == "unzip": + batch = unzip(arg, Progress("Unpacking")) + res = validate(batch, Progress("Validating")) + result = f"unpacked into {batch}: {res['state']}" + else: + raise ValueError(f"unknown job {kind}") + except Exception as ex: + _job_update(state="error", finished=uc.now(), error=str(ex)[:600], step="") + raise + st = uc.read_json(_path("job.json"), {}) or {} + _job_update(state="done", finished=uc.now(), result=result, step="", current="", + done_bytes=st.get("total_bytes") or 0) + return result + + +class _Busy(RuntimeError): + pass + + +class _Lock: + """Exclusive lock around anything that builds or swaps BPRT, shared by the + jobs, the inline rollback and recover().""" + + def __enter__(self): + self.fd = os.open(_path(".lock"), os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(self.fd) + raise _Busy("another BPRT activation or rollback is running") from None + return self + + def __exit__(self, *exc): + fcntl.flock(self.fd, fcntl.LOCK_UN) + os.close(self.fd) + return False + + +# --- incoming batches -------------------------------------------------------- + +def _read_sync(folder): + path = os.path.join(folder, "sync.json") + try: + with open(path, encoding="utf-8-sig") as f: + data = json.load(f) + except OSError: + return None + except ValueError as ex: + raise ValueError(f"sync.json is not valid JSON: {ex}") from None + if not isinstance(data, dict): + raise ValueError("sync.json is not a JSON object") + return data + + +def _list(value): + """A JSON list from sync.json. Windows PowerShell 5.1's ConvertTo-Json + writes an array that carries PowerShell's extra type data as + {"value": [...], "Count": n}; take the list out of that too.""" + if isinstance(value, dict) and isinstance(value.get("value"), list): + value = value["value"] + return value if isinstance(value, list) else [] + + +def validation(name): + return uc.read_json(os.path.join(_state("batches"), f"{check_batch(name)}.json"), None) + + +def batches(): + """Incoming batches, newest first: name, files, bytes, sync.json facts and + the last validation. A '.part' folder is an sftp upload still + running (Sync-BPRT.ps1 renames it when done).""" + root = incoming_root() + out = [] + if not os.path.isdir(root): + return out + live_files = None + for name in os.listdir(root): + folder = os.path.join(root, name) + if name.startswith(".") or not os.path.isdir(folder) or os.path.islink(folder): + continue + uploading = name.endswith(".part") + base = name[:-5] if uploading else name + if not _BATCH.match(base): + continue + files, _ = walk(folder) + row = {"name": name, "uploading": uploading, + "files": len([f for f in files if f != "sync.json"]), + "bytes": sum(st.st_size for f, st in files.items() if f != "sync.json"), + "modified": max([st.st_mtime for st in files.values()] or [os.stat(folder).st_mtime]), + "sync": None, "sync_error": "", "validation": None} + if not uploading: + try: + sync = _read_sync(folder) + if sync: + row["sync"] = {k: sync.get(k) for k in ("computer", "user", "created", "source")} + row["sync"]["listed"] = len(_list(sync.get("files"))) + except ValueError as ex: + row["sync_error"] = str(ex) + res = validation(name) + if res: + if live_files is None: + live_files = walk(bprt_dir())[0] + res["stale"] = _stale(res, live_files) + row["validation"] = res + out.append(row) + out.sort(key=lambda r: r["modified"], reverse=True) + return out + + +def _snapshot(files): + """{relpath: [size, mtime_ns]} of a walk(), as a validation records it.""" + return {rel: [st.st_size, st.st_mtime_ns] for rel, st in files.items()} + + +def _live_moved(saved, files): + """What differs between a recorded snapshot of the live BPRT and its + files now, or ''. Any file added, removed or changed there (an MCL upload + on /enrollment, an edit over SMB) means the validated plan, which decides + what the new folder keeps, is out of date.""" + for rel in sorted(set(saved) | set(files), key=str.lower): + if rel not in files: + return f"{rel} is gone from the live BPRT" + if rel not in saved: + return f"{rel} was added to the live BPRT" + st = files[rel] + if [st.st_size, st.st_mtime_ns] != list(saved[rel]): + return f"{rel} in the live BPRT changed" + return "" + + +def _stale(res, live_files=None): + """Why a validation no longer holds (a file changed since), or ''. + live_files: a walk() of the live BPRT, when the caller has one.""" + try: + folder = batch_dir(res["batch"]) + except ValueError: + return "bad batch name" + if res.get("live_all") is not None: + if live_files is None: + live_files = walk(bprt_dir())[0] + why = _live_moved(res["live_all"], live_files) + if why: + return f"{why} since the batch was validated" + for path, (size, mtime) in (res.get("batch_stat") or {}).items(): + try: + st = os.lstat(os.path.join(folder, *path.split("/"))) + except OSError: + return f"{path} is gone from the batch" + if (st.st_size, st.st_mtime_ns) != (size, mtime): + return f"{path} in the batch changed since it was validated" + for rel, (size, mtime) in (res.get("live_stat") or {}).items(): + try: + st = os.lstat(os.path.join(bprt_dir(), *rel.split("/"))) + except OSError: + return f"{rel} is gone from the live BPRT" + if (st.st_size, st.st_mtime_ns) != (size, mtime): + return f"{rel} in the live BPRT changed since the batch was validated" + return "" + + +def _check_rel(path): + """A relative path from sync.json, or ValueError: forward slashes, no + empty, '.' or '..' parts, nothing Windows cannot name.""" + if not isinstance(path, str) or not path or path.startswith("/") or len(path) > 240: + raise ValueError(f"bad path {path!r}") + parts = path.split("/") + for p in parts: + if p in ("", ".", "..") or _WIN_BAD.search(p) or p != p.rstrip(" ."): + raise ValueError(f"bad path {path!r}") + return path + + +def _dest(vpath): + """Name a vendor file gets in BPRT. Case-insensitive, as on the PCs: a + vendor select-pcconfig.ps1 next to our Select-PCConfig.ps1 would be one + file to Windows.""" + for vendor, kept in VENDOR_RENAMED.items(): + if vpath.lower() == vendor.lower(): + return kept + return vpath + + +def _ppkg_names(names): + """Top-level names the enroll scripts may treat as packages.""" + return [n for n in names if "/" not in n and (n.lower().endswith(".ppkg") or mcl_ppkg.is_mcl_name(n))] + + +def _versions(names): + """{prefix: file name} for the vendor-valid MCL packages among names.""" + out = {} + for n in names: + if mcl_ppkg.is_mcl_name(n): + out.setdefault(mcl_ppkg.parse(n, strict=False)["prefix"], []).append(n) + return out + + +def check_packages(names, errors, warnings): + """The vendor rules on the package names a folder will hold.""" + pk = _ppkg_names(names) + mcl = [n for n in pk if mcl_ppkg.is_mcl_name(n)] + for n in pk: + if n not in mcl: + warnings.append(f"{n} does not match the vendor's package naming; the enroll scripts ignore it") + elif "_exp_xxxxxxxx_" not in n.lower(): + warnings.append(f"{n} has no Exp_XXXXXXXX placeholder: a package already used on a PC, " + "the enroll script cannot inject a new token into it") + if not mcl: + errors.append("no MCL package (*_v..ppkg) in the new folder") + seen = set() + for n, others in sorted(mcl_ppkg._clashes(mcl).items()): + prefix = mcl_ppkg.parse(n, strict=False)["prefix"] + group = frozenset([n, *others]) + if group in seen: + continue # the same pair, seen from its other package + seen.add(group) + errors.append(f"{prefix} would match {n} and {', '.join(others)}: the enroll scripts fail " + "when one tenant/environment/purpose prefix finds two packages") + return mcl + + +def validate(name, progress=None): + """Check an incoming batch and work out what activating it would change. + Saves and returns the result (state 'valid' or 'invalid').""" + folder = batch_dir(name) + progress = progress or Progress() + res = {"batch": name, "validated": uc.now(), "state": "invalid", "errors": [], + "warnings": [], "sync": {}, "plan": [], "batch_stat": {}, "live_stat": {}} + errors, warnings = res["errors"], res["warnings"] + try: + _validate(folder, res, progress) + except ValueError as ex: + errors.append(str(ex)) + if not errors: + res["state"] = "valid" + uc.write_json(os.path.join(_state("batches"), f"{name}.json"), res) + return res + + +def _validate(folder, res, progress): + errors, warnings = res["errors"], res["warnings"] + if not os.path.isdir(folder): + raise ValueError("the batch folder is gone") + sync = _read_sync(folder) + if sync is None: + raise ValueError("no sync.json in the batch: Sync-BPRT.ps1 uploads it last, so the upload " + "did not finish (or this folder was made by hand)") + res["sync"] = {k: sync.get(k) for k in ("computer", "user", "created", "source", "tool")} + listed = _list(sync.get("files")) + if not isinstance(listed, list) or not listed: + raise ValueError("sync.json lists no files") + want, seen_lc = {}, {} + for row in listed: + if not isinstance(row, dict): + raise ValueError("sync.json: every file entry must be an object") + path = _check_rel(row.get("path")) + if path.lower() in seen_lc: + raise ValueError(f"sync.json lists {path} twice (names differing only by case " + "are one file on Windows)") + seen_lc[path.lower()] = path + try: + size = int(row.get("size")) + except (TypeError, ValueError): + raise ValueError(f"sync.json: {path} has no size") from None + sha = str(row.get("sha256") or "").upper() + if not re.fullmatch(r"[0-9A-F]{64}", sha): + raise ValueError(f"sync.json: {path} has no sha256") + want[path] = {"size": size, "sha256": sha} + for e in _list(sync.get("parse_errors")): + if isinstance(e, dict): + errors.append(f"{e.get('path')}: PowerShell could not parse it on the work PC " + f"({str(e.get('error', ''))[:200]}); the MediaCreator cache may be corrupt") + claimed = set(x for x in _list(sync.get("uploaded")) if isinstance(x, str)) + + have, _ = walk(folder) + have.pop("sync.json", None) + root = bprt_dir() + if os.path.islink(root) or not os.path.isdir(root): + raise ValueError(f"{root} is missing or not a folder") + live, skipped = walk(root) + res["live_all"] = _snapshot(live) + view = vendor_view(root, live) + # Windows names are case-insensitive: the work PC sees a file the vendor + # renamed by case only as unchanged and does not upload it, so a listed + # path finds its live file case-insensitively (exact spelling first). + view_lc = {vp.lower(): vp for vp in sorted(view, reverse=True)} + want_lc = _lc(want) + cache = HashCache() + + def live_match(path): + return path if path in view else view_lc.get(path.lower()) + + # Hash what was uploaded and what in the live folder is not cached yet. + todo_live = [rel for vp, rel in view.items() if vp.lower() in want_lc and not cache.get(rel, live[rel])] + progress.start(sum(st.st_size for st in have.values()) + + sum(live[r].st_size for r in todo_live), len(have) + len(todo_live)) + got = {} + for path in sorted(have): + progress.current(path) + st = have[path] + sha, nobom = hash_file(os.path.join(folder, *path.split("/")), path, progress.add) + got[path] = sha + res["batch_stat"][path] = [st.st_size, st.st_mtime_ns] + if path in want and sha == want[path]["sha256"]: + # Pre-seed the cache under the name it will have in BPRT. + cache.put(_dest(path), st, sha, nobom) + for path in sorted(set(have) - set(want)): + warnings.append(f"{path} was uploaded but is not in sync.json; it is ignored") + for path in sorted(claimed - set(have)): + if path in want: + errors.append(f"{path}: sync.json says it was uploaded but it is not in the batch") + + plan, diff = [], {"new": [], "changed": [], "unchanged": [], "removed": []} + matched = set() + for path in sorted(want, key=str.lower): + w = want[path] + dest = _dest(path) + vp = live_match(path) + if vp is not None: + matched.add(vp) + if vp != path: + warnings.append(f"{vp} is now spelled {path} (a change of case only); the new BPRT " + "uses the new spelling") + if path in have: + if have[path].st_size != w["size"] or got[path] != w["sha256"]: + errors.append(f"{path}: hash mismatch (uploaded {have[path].st_size} bytes, sha256 " + f"{got[path][:12]}..., sync.json says {w['size']} bytes, " + f"{w['sha256'][:12]}...); upload it again") + continue + item = {"path": path, "dest": dest, "source": "incoming", "size": w["size"]} + if _is_script(path) and _bom_less_non_ascii(os.path.join(folder, *path.split("/"))): + item["bom_fix"] = True + if dest == VENDOR_RENAMED.get("Select-PCConfig.ps1") and is_shim( + os.path.join(folder, *path.split("/"))): + errors.append(f"{path} in the batch is our shim, not the vendor's dialog: kept as " + f"{dest} it would call itself. Upload the vendor folder, not the server's") + plan.append(item) + rel = view.get(vp) if vp else None + if rel is None: + diff["new"].append(path) + else: + progress.current(path) + e = cache.hash(root, rel, live[rel], progress.add) + res["live_stat"][rel] = [live[rel].st_size, live[rel].st_mtime_ns] + diff["unchanged" if _same(e, w["sha256"]) else "changed"].append(path) + else: + rel = view.get(vp) if vp else None + if rel is None: + errors.append(f"{path} was not uploaded and is not in the live BPRT") + continue + progress.current(path) + e = cache.hash(root, rel, live[rel], progress.add) + if not _same(e, w["sha256"]): + errors.append(f"{path} was not uploaded, but the live copy differs " + f"(live {e['sha256'][:12]}..., sync.json {w['sha256'][:12]}...); " + "run Sync-BPRT.ps1 again") + continue + res["live_stat"][rel] = [live[rel].st_size, live[rel].st_mtime_ns] + plan.append({"path": path, "dest": dest, "source": "live", "from": rel, "size": w["size"]}) + diff["unchanged"].append(path) + diff["removed"] = sorted((vp for vp in view if vp not in matched), key=str.lower) + cache.save() + for s in skipped: + warnings.append(f"{s} in the live BPRT is a symlink or not a regular file; the new BPRT " + "will not have it (it stays in the archive)") + + # The vendor rules and the files the flow calls by path. + check_packages(list(want), errors, warnings) + for path in sorted(have): + if path in want and path.lower().endswith(".ppkg"): + with open(os.path.join(folder, *path.split("/")), "rb") as f: + if f.read(len(mcl_ppkg.WIM_MAGIC)) != mcl_ppkg.WIM_MAGIC: + errors.append(f"{path} is not a provisioning package (no WIM header)") + for r in REQUIRED: + if r.lower() not in want_lc: + errors.append(f"{r} is missing: the bulk-enroll flow calls it by path") + for name in OVERLAY: + if name.lower() in _lc(want) and name not in VENDOR_RENAMED: + warnings.append(f"the vendor now ships {name}; the site copy replaces it as before") + for kept in VENDOR_RENAMED.values(): + if kept.lower() in _lc(want): + errors.append(f"sync.json lists {kept}, the name the vendor's copy is kept under here") + srcs = overlay_sources(root, live) + if "Select-PCConfig.ps1" not in srcs: + errors.append("our Select-PCConfig.ps1 shim is not in the live BPRT (and no repo copy is " + "here): copy playbook/shopfloor-setup/BPRT/Select-PCConfig.ps1 there first") + for name in OVERLAY_REQUIRED: + if name not in srcs: + warnings.append(f"{name} is not in the live BPRT, so the new one will not have it " + "either: standard and engineer specialize (Order 39) calls it") + for path in diff["removed"]: + if path.lower().endswith(".zip"): + warnings.append(f"{path} is no longer shipped; MCL packages that install from it would fail") + + old_pk = _versions(view) + new_pk = _versions(want) + ppkg = [] + for prefix in sorted(set(old_pk) | set(new_pk), key=str.lower): + olds, news = old_pk.get(prefix, []), new_pk.get(prefix, []) + if sorted(olds) != sorted(news): + ppkg.append({"prefix": prefix, + "old": [{"name": n, "version": mcl_ppkg.parse(n, strict=False)["version"]} for n in olds], + "new": [{"name": n, "version": mcl_ppkg.parse(n, strict=False)["version"]} for n in news]}) + zips = [] + for path in diff["new"] + diff["changed"]: + if path.lower().endswith(".zip"): + vp = live_match(path) + rel = view[vp] if vp else None + zips.append({"path": path, "old_size": live[rel].st_size if rel else None, + "new_size": want[path]["size"]}) + + res["plan"] = plan + res["diff"] = diff + res["counts"] = {k: len(v) for k, v in diff.items()} + res["ppkg"] = ppkg + res["zips"] = zips + res["bom_fix"] = [i["path"] for i in plan if i.get("bom_fix")] + res["upload_bytes"] = sum(st.st_size for st in have.values()) + res["total_bytes"] = sum(w["size"] for w in want.values()) + + +# --- activation -------------------------------------------------------------- + +def _link(src, dst): + """Hard link src at dst; a copy when the filesystem will not link.""" + try: + os.link(src, dst) + except OSError as ex: + if ex.errno not in (errno.EXDEV, errno.EPERM, errno.EMLINK, errno.ENOTSUP, errno.EACCES): + raise + shutil.copy2(src, dst) + + +def _makedirs(root, rel_dir, live_root): + """Create root/rel_dir level by level with the owner and mode of the same + folder in the live BPRT (else of BPRT itself).""" + if not rel_dir: + return + cur, live_cur = root, live_root + for part in rel_dir.split("/"): + cur = os.path.join(cur, part) + live_cur = os.path.join(live_cur, part) + if not os.path.isdir(cur): + os.mkdir(cur) + try: + mode = stat.S_IMODE(os.stat(live_cur).st_mode) + except OSError: + mode = _owner()[2] or 0o755 + _match_owner(cur, mode) + + +def _file_mode(live_files, rel): + """Mode for a new file: the one it replaces, else the most common one in + the live folder.""" + st = live_files.get(rel) + if st: + return stat.S_IMODE(st.st_mode) + modes = Counter(stat.S_IMODE(s.st_mode) for s in live_files.values()) + return modes.most_common(1)[0][0] if modes else _DEFAULT_FILE_MODE + + +def _unique(root, base): + path = os.path.join(root, base) + n = 1 + while os.path.exists(path): + n += 1 + path = os.path.join(root, f"{base}-{n}") + return path + + +def recover(): + """Finish or undo a swap a crash interrupted between its two renames, so + BPRT is never left missing. Returns what was done, or ''. Takes the lock: + while a job or rollback holds it, swap.json describes a swap in progress + (BPRT may be missing for a moment), not a crashed one, so it is left + alone; the holder cleans up itself.""" + try: + with _Lock(): + return _recover() + except _Busy: + return "" + + +def _recover(): + """recover() for a caller that already holds the lock.""" + intent = uc.read_json(_path("swap.json"), None) + if not intent: + return "" + live = bprt_dir() + done = "" + if not os.path.exists(live): + if intent.get("new") and os.path.isdir(intent["new"]): + os.rename(intent["new"], live) + done = f"finished an interrupted swap: {os.path.basename(intent['new'])} is now BPRT" + elif intent.get("archive") and os.path.isdir(intent["archive"]): + os.rename(intent["archive"], live) + done = f"undid an interrupted swap: {os.path.basename(intent['archive'])} is BPRT again" + try: + os.remove(_path("swap.json")) + except OSError: + pass + if done: + _audit("BPRT_RECOVER", done) + return done + + +def _swap(src): + """BPRT to BPRT-archive/full-, then src to BPRT. Returns the archive + path. SIGTERM (systemctl stop) waits until both renames are done.""" + live = bprt_dir() + os.makedirs(archive_root(), exist_ok=True) + dest = _unique(archive_root(), f"full-{_stamp()}") + uc.write_json(_path("swap.json"), {"new": src, "archive": dest, "at": uc.now()}) + old = None + try: + old = signal.signal(signal.SIGTERM, signal.SIG_IGN) + except ValueError: + pass # not the main thread (the webapp's inline rollback) + try: + os.rename(live, dest) + try: + os.rename(src, live) + except OSError: + os.rename(dest, live) + raise + finally: + if old is not None: + signal.signal(signal.SIGTERM, old) + if os.path.isdir(live): + try: + os.remove(_path("swap.json")) + except OSError: + pass + # else both renames failed after the first: swap.json stays, so + # recover() puts BPRT back on the next page load or job. + return dest + + +def activate(name, progress=None): + """Build BPRT.new from the validated batch and swap it in.""" + progress = progress or Progress() + with _Lock(): + _recover() + res = validation(name) + if not res or res.get("state") != "valid": + raise RuntimeError("this batch is not validated") + folder = batch_dir(name) + live = bprt_dir() + if os.path.islink(live) or not os.path.isdir(live): + raise RuntimeError(f"{live} is missing or not a folder") + live_files, _ = walk(live) + why = _stale(res, live_files) + if why: + raise RuntimeError(f"{why}; validate it again") + srcs = overlay_sources(live, live_files) + if "Select-PCConfig.ps1" not in srcs: + raise RuntimeError("our Select-PCConfig.ps1 shim is not in the live BPRT") + new = new_dir() + if os.path.lexists(new): + shutil.rmtree(new) # left by a failed run; never live + os.mkdir(new) + _match_owner(new, _owner()[2] or 0o755) + try: + progress.step("Building BPRT.new") + progress.start(len(res["plan"])) + for item in res["plan"]: + dest_rel = item["dest"] + dest = os.path.join(new, *dest_rel.split("/")) + _makedirs(new, dest_rel.rpartition("/")[0], live) + if item["source"] == "live": + _link(os.path.join(live, *item["from"].split("/")), dest) + else: + src = os.path.join(folder, *item["path"].split("/")) + if item.get("bom_fix"): + with open(src, "rb") as f: + data = f.read() + with open(dest, "wb") as f: + f.write(BOM + data) + shutil.copystat(src, dest) + else: + _link(src, dest) + _match_owner(dest, _file_mode(live_files, dest_rel)) + if item["source"] == "live": + expect = res["live_stat"][item["from"]][0] # BOM-fixed earlier: 3 bytes more + else: + expect = item["size"] + (len(BOM) if item.get("bom_fix") else 0) + if os.path.getsize(dest) != expect: + raise RuntimeError(f"{dest_rel} has the wrong size in BPRT.new") + progress.add(1) + progress.step("Applying the site overlay") + top = os.listdir(new) + for oname, src in srcs.items(): + dest = os.path.join(new, oname) + for n in top: + if n.lower() == oname.lower(): + os.remove(os.path.join(new, n)) # the vendor shipped one (any case); ours wins + if src.startswith(live + os.sep): + _link(src, dest) + else: + shutil.copy2(src, dest) # the repo copy of our shim + _match_owner(dest, _file_mode(live_files, oname)) + # Last look at the folder that is about to go live. + names, _ = walk(new) + errors = [] + check_packages(list(names), errors, []) + missing = [r for r in REQUIRED if _dest(r).lower() not in _lc(names)] + if missing: + errors.append(f"missing {', '.join(missing)}") + if errors: + raise RuntimeError("BPRT.new failed its final check: " + "; ".join(errors)) + # An MCL upload on /enrollment (or an SMB edit) while BPRT.new + # was built would be lost with the old folder: check again. + why = _live_moved(_snapshot(live_files), walk(live)[0]) + if why: + raise RuntimeError(f"{why} while BPRT.new was built; nothing was replaced, " + "validate the batch again") + progress.step("Swapping BPRT.new in") + archive = _swap(new) + except BaseException: + if os.path.isdir(live): # never drop BPRT.new while it is the only full copy + shutil.rmtree(new, ignore_errors=True) + raise + progress.step("Recording") + entry = {"id": _stamp(), "at": uc.now(), "kind": "activate", "batch": name, + "archive": os.path.basename(archive), "sync": res.get("sync", {}), + "counts": res.get("counts", {}), "ppkg": res.get("ppkg", []), + "bom_fix": res.get("bom_fix", [])} + _add_history(entry) + shutil.rmtree(folder, ignore_errors=True) + try: + os.remove(os.path.join(_state("batches"), f"{name}.json")) + except OSError: + pass + try: + manifest(build=True) # seeds the cache for the BOM-fixed files + except Exception: + pass + _audit("BPRT_ACTIVATE", f"{name}: {res['counts']}; previous BPRT in {entry['archive']}") + return entry + + +# --- archives and rollback --------------------------------------------------- + +def archives(): + """Full BPRT copies in BPRT-archive, newest first, with the bytes only + they hold (files not hard-linked into the live BPRT).""" + root = archive_root() + out = [] + if not os.path.isdir(root): + return out + live_inodes = {(st.st_dev, st.st_ino) for st in walk(bprt_dir())[0].values()} + for name in os.listdir(root): + folder = os.path.join(root, name) + if not _ARCHIVE.match(name) or not os.path.isdir(folder) or os.path.islink(folder): + continue + files, _ = walk(folder) + view = vendor_view(folder, files) + out.append({ + "name": name, "count": len(files), + "bytes": sum(st.st_size for st in files.values()), + "own_bytes": sum(st.st_size for st in files.values() + if (st.st_dev, st.st_ino) not in live_inodes), + "modified": os.stat(folder).st_mtime, + "packages": sorted(n for n in files if mcl_ppkg.is_mcl_name(n)), + "missing_required": [r for r in REQUIRED if r not in view], + }) + out.sort(key=lambda a: a["name"], reverse=True) + return out + + +def _check_archive(name): + if not _ARCHIVE.match(str(name or "")): + raise ValueError(f"{name or '(blank)'} is not a BPRT archive") + folder = os.path.join(archive_root(), name) + if not os.path.isdir(folder) or os.path.islink(folder): + raise ValueError(f"{name} is not in BPRT-archive") + return folder + + +def rollback(name): + """Make an archived BPRT live again; the current one is archived first, + so a rollback can itself be rolled back.""" + folder = _check_archive(name) + if busy(): + raise RuntimeError(f"a BPRT job ({busy()}) is running") + with _Lock(): + _recover() + files, _ = walk(folder) + errors = [] + check_packages(list(files), errors, []) + if errors: + raise ValueError("; ".join(errors)) + archive = _swap(folder) + entry = {"id": _stamp(), "at": uc.now(), "kind": "rollback", "restored": name, + "archive": os.path.basename(archive)} + _add_history(entry) + return entry + + +def delete_archive(name): + folder = _check_archive(name) + if busy(): + raise RuntimeError(f"a BPRT job ({busy()}) is running") + with _Lock(): + shutil.rmtree(folder) + return name + + +def delete_batch(name): + folder = os.path.join(incoming_root(), str(name or "")) + base = name[:-5] if str(name).endswith(".part") else name + check_batch(base) + if not os.path.isdir(folder) or os.path.islink(folder): + raise ValueError(f"{name} is not an incoming batch") + st = job_state() + if st.get("running") and st.get("arg") == base: + raise RuntimeError(f"the {st.get('kind')} job is using this batch") + shutil.rmtree(folder) + if base == name: # X.part is an upload; the validation of a finished X stays + try: + os.remove(os.path.join(_state("batches"), f"{base}.json")) + except OSError: + pass + return name + + +def history(): + return uc.read_json(_path("history.json"), []) or [] + + +def _add_history(entry): + hist = history() + hist.insert(0, entry) + uc.write_json(_path("history.json"), hist[:100]) + + +# --- zip upload fallback ----------------------------------------------------- + +def clean_zip_name(name): + name = os.path.basename(str(name or "").replace("\\", "/")).strip() + if not name.lower().endswith(".zip") or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9 ._()+-]{0,150}", name): + raise ValueError("upload a .zip file (letters, digits, spaces, . _ - ( ) + in the name)") + return name + + +def save_zip(stream, name, length=None): + """Stream an uploaded zip to BPRT-incoming/.uploads (same filesystem as + BPRT, never in memory). Returns (name, bytes).""" + name = clean_zip_name(name) + ensure_incoming() + folder = uploads_dir() + os.makedirs(folder, exist_ok=True) + free = shutil.disk_usage(folder).free + if length and length * 2 > free - (1 << 30): + raise RuntimeError(f"not enough disk space: the zip and its unpacked files need about " + f"{2 * length // (1 << 20)} MB, {free // (1 << 20)} MB free") + part = os.path.join(folder, f".{name}.{os.getpid()}.part") + written = 0 + try: + with open(part, "wb") as f: + while True: + block = stream.read(1 << 20) + if not block: + break + f.write(block) + written += len(block) + if length and written != length: + raise RuntimeError(f"upload incomplete: {written} of {length} bytes") + if not written: + raise RuntimeError("empty upload") + if not zipfile.is_zipfile(part): + raise ValueError(f"{name} is not a zip file") + os.replace(part, os.path.join(folder, name)) + finally: + if os.path.exists(part): + os.remove(part) + return name, written + + +def uploaded_zips(): + folder = uploads_dir() + if not os.path.isdir(folder): + return [] + out = [] + for n in sorted(os.listdir(folder)): + p = os.path.join(folder, n) + if n.lower().endswith(".zip") and not n.startswith(".") and os.path.isfile(p): + st = os.stat(p) + out.append({"name": n, "size": st.st_size, "modified": st.st_mtime}) + return out + + +def delete_zip(name): + path = os.path.join(uploads_dir(), clean_zip_name(name)) + if not os.path.isfile(path): + raise ValueError(f"{name} is not an uploaded zip") + os.remove(path) + + +def _zip_members(zf): + """[(ZipInfo, relpath)] of the files in a zip, any single top folder + (e.g. BPRT/) stripped; ValueError on unsafe names.""" + rows = [] + for info in zf.infolist(): + if info.is_dir(): + continue + name = info.filename.replace("\\", "/") + if name.startswith("__MACOSX/") or name.rsplit("/", 1)[-1] in ("Thumbs.db", ".DS_Store"): + continue + if name.startswith("/") or re.match(r"^[A-Za-z]:", name): + raise ValueError(f"zip entry {info.filename!r} has an absolute path") + rows.append([info, name]) + tops = {r[1].split("/", 1)[0] for r in rows} + if len(tops) == 1 and all("/" in r[1] for r in rows): + cut = len(next(iter(tops))) + 1 + for r in rows: + r[1] = r[1][cut:] + for r in rows: + _check_rel(r[1]) + return [(info, rel) for info, rel in rows] + + +def unzip(name, progress=None): + """Unpack an uploaded zip into a new incoming batch. A zip made by + Sync-BPRT.ps1 -ZipOut carries its sync.json; any other zip is taken as the + complete vendor folder and gets a sync.json listing everything in it.""" + progress = progress or Progress() + name = clean_zip_name(name) + src = os.path.join(uploads_dir(), name) + if not os.path.isfile(src): + raise ValueError(f"{name} is not an uploaded zip") + base = f"{_stamp()}-zip" + final = _unique(incoming_root(), base) + part = final + ".part" + os.makedirs(part) + try: + with zipfile.ZipFile(src) as zf: + members = _zip_members(zf) + if not members: + raise ValueError(f"{name} holds no files") + need = sum(i.file_size for i, _ in members) + free = shutil.disk_usage(incoming_root()).free + if need > free - (1 << 30): # keep 1 GB for the share + raise RuntimeError(f"not enough disk space: {name} unpacks to {need // (1 << 20)} MB, " + f"{free // (1 << 20)} MB free") + progress.start(sum(i.file_size for i, _ in members), len(members)) + listed = [] + for info, rel in members: + progress.current(rel) + dest = os.path.join(part, *rel.split("/")) + os.makedirs(os.path.dirname(dest), exist_ok=True) + h = hashlib.sha256() + with zf.open(info) as fin, open(dest, "wb") as fout: + for block in iter(lambda: fin.read(1 << 22), b""): + fout.write(block) + h.update(block) + progress.add(len(block)) + if rel != "sync.json": + listed.append({"path": rel, "size": info.file_size, "sha256": h.hexdigest().upper()}) + if not os.path.exists(os.path.join(part, "sync.json")): + uc.write_json(os.path.join(part, "sync.json"), { + "format": 1, "tool": "webapp zip upload", "source": name, + "computer": "", "user": "", "created": uc.now(), + "files": listed, "uploaded": [r["path"] for r in listed], + }) + os.rename(part, final) + except BaseException: + shutil.rmtree(part, ignore_errors=True) + raise + os.remove(src) # unpacked; the zip would only double the space used + _audit("BPRT_UNZIP", f"{name} into {os.path.basename(final)}") + return os.path.basename(final) + + +# --- imaging in progress ----------------------------------------------------- + +def _epoch(iso): + try: + dt = datetime.fromisoformat(str(iso)) + except (TypeError, ValueError): + return None + if not dt.tzinfo: + dt = dt.replace(tzinfo=timezone.utc) + return dt.timestamp() + + +def active_imaging(): + """Imaging sessions still running (pushed or inferred from the logs in the + last few hours): each of those PCs copies BPRT while it images.""" + try: + from services import imaging_status + sessions = imaging_status.list_sessions() + window = imaging_status.ACTIVE_WINDOW_HOURS * 3600 + except Exception: + return [] + cutoff = time.time() - window + out = [] + for s in sessions: + if s.get("status") in ("succeeded", "failed"): + continue + ts = _epoch(s.get("last_updated")) + if ts is None or ts < cutoff: + continue + out.append({"id": s.get("serial") or s.get("mac") or s.get("ip") or "?", + "hostname": s.get("hostname") or "", "stage": s.get("current_stage") or "", + "last_updated": s.get("last_updated")}) + return out + + +def sync_script(): + """Path of scripts/windows/Sync-BPRT.ps1 when the webapp can serve it.""" + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + for p in (os.environ.get("BPRT_SYNC_SCRIPT", ""), + os.path.join(here, "static", "Sync-BPRT.ps1"), + os.path.join(here, "..", "scripts", "windows", "Sync-BPRT.ps1")): + if p and os.path.isfile(p): + return os.path.abspath(p) + return None + + +def _audit(action, detail): + try: + from services.audit import audit + audit(action, detail) + except Exception: + pass # audit log not writable (local tests): history.json still has it + + +# --- job entry point --------------------------------------------------------- + +def main(argv): + if not argv or argv[0] not in ("manifest", "validate", "activate", "unzip"): + print("usage: python3 -m services.bprt_sync manifest | validate BATCH | activate BATCH | unzip NAME", + file=sys.stderr) + return 2 + print(run_job(argv[0], argv[1] if len(argv) > 1 else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/webapp/services/control_files.py b/webapp/services/control_files.py new file mode 100644 index 0000000..b7ec43f --- /dev/null +++ b/webapp/services/control_files.py @@ -0,0 +1,478 @@ +"""Safe edits of an image's control files: preview, backup, atomic write, undo. + +Every save on the image pages (Drivers, Unattend, OS and Packages) follows +the same three steps, and this module is the shared implementation: + + 1. plan_json(path, data) / plan_text(path, text) render what WOULD be + written, in the file's own on-disk format, and diff it against the + current file. Nothing is written. The page shows plan["diff"]. + 2. commit(image_type, scope, plans, action) backs every changed file up + next to itself as .bak--webui, writes the new + text atomically (temp file in the same dir + os.replace, original + owner and mode kept) and remembers the save as the undo point for + (image_type, scope). Pass expect={path: sha256} (from the preview) to + refuse the save when a file changed after the preview was shown. + 3. undo(image_type, scope) puts the backups of the last commit back, but + only while every file still holds exactly what that commit wrote. + +The on-disk format is never changed by a save: JSON keeps its indent, BOM, +line endings, trailing newline and ASCII escaping (detect_json_format). +A file that does not round-trip byte for byte (a hand-edited vendor file +with mixed tabs and spaces, say) is not re-rendered whole: for a top-level +JSON list the entries that did not change keep their exact original text +and only new or edited entries are rendered (_splice_list), so the diff +shows the real change. Only when that is impossible does a plan say +"reformat": True, so the page can warn that whitespace will move. + +commit() and undo() hold one lock (UPDATES_DIR/webui-undo/.lock) from the +stale check to the last write, so two saves cannot interleave, and a +commit that fails half way puts the files it already wrote back. + +Undo records live in UPDATES_DIR/webui-undo/--.json; only +the last save per scope can be undone (the .bak files stay on the share +for anything older). +""" + +import contextlib +import difflib +import fcntl +import hashlib +import json +import os +import re +import shutil +import tempfile +from datetime import datetime + +from services import updates_common as uc + +BACKUP_TAG = "webui" +_BOM = "\ufeff" + + +class Conflict(Exception): + """A file changed on disk since the preview or since the last save.""" + + +# --- format ------------------------------------------------------------------ + +def encode(text): + """Bytes as written: UTF-8, a leading \\ufeff becomes a real BOM.""" + if text.startswith(_BOM): + return b"\xef\xbb\xbf" + text[1:].encode("utf-8") + return text.encode("utf-8") + + +def sha256_text(text): + return hashlib.sha256(encode(text)).hexdigest() + + +def file_sha256(path): + """sha256 of the file's bytes ("" hashed when missing).""" + try: + with open(path, "rb") as fh: + return hashlib.sha256(fh.read()).hexdigest() + except OSError: + return hashlib.sha256(b"").hexdigest() + + +def read_text(path): + """Current file content as text ("" when missing). A BOM is kept as the + leading \\ufeff character so it survives a round trip.""" + try: + with open(path, "rb") as fh: + raw = fh.read() + except OSError: + return "" + if raw.startswith(b"\xef\xbb\xbf"): + return _BOM + raw[3:].decode("utf-8", errors="replace") + return raw.decode("utf-8", errors="replace") + + +def detect_json_format(text): + """How an existing JSON file is laid out. Defaults (for a new file) are + indent 4, no BOM, LF, trailing newline, non-ASCII kept as is.""" + fmt = {"indent": 4, "bom": False, "newline": "\n", "trailing_newline": True, + "ensure_ascii": False} + if not text: + return fmt + fmt["bom"] = text.startswith(_BOM) + body = text[1:] if fmt["bom"] else text + fmt["newline"] = "\r\n" if "\r\n" in body else "\n" + fmt["trailing_newline"] = body.endswith("\n") + lines = body.splitlines() + if len(lines) < 2: + # One line: compact JSON, unless it is an empty "[]" / "{}", which + # says nothing about layout (keep the indent 4 default). + if not re.fullmatch(r"\s*(\[\s*\]|\{\s*\})\s*", body): + fmt["indent"] = None + else: + m = re.match(r"[ \t]+", lines[1]) + if m: + ws = m.group(0) + fmt["indent"] = "\t" if ws.startswith("\t") else len(ws) + # \uXXXX escapes and no raw non-ASCII: the writer escaped everything. + fmt["ensure_ascii"] = bool(re.search(r"\\u[0-9a-fA-F]{4}", body)) and body.isascii() + return fmt + + +def render_json(data, fmt): + text = json.dumps(data, indent=fmt["indent"], ensure_ascii=fmt["ensure_ascii"]) + if fmt["trailing_newline"]: + text += "\n" + if fmt["newline"] != "\n": + text = text.replace("\n", fmt["newline"]) + if fmt["bom"]: + text = _BOM + text + return text + + +# --- plan ---------------------------------------------------------------------- + +def _diff(old, new, name): + return "".join(difflib.unified_diff( + old.splitlines(keepends=True), new.splitlines(keepends=True), + fromfile=f"a/{name}", tofile=f"b/{name}", n=3)) + + +def plan_text(path, new_text, label=None): + """What saving new_text to path would change. Keys: path, name, old, + new, changed, diff, sha_before, existed, reformat (always False here).""" + old = read_text(path) + name = label or os.path.basename(path) + return { + "path": path, + "name": name, + "old": old, + "new": new_text, + "changed": old != new_text, + "diff": _diff(old, new_text, name) if old != new_text else "", + "sha_before": file_sha256(path), + "existed": os.path.isfile(path), + "reformat": False, + } + + +_WS = re.compile(r"[ \t\r\n]*") + + +def _key(obj): + """Order-sensitive identity of a JSON value (key order matters on disk).""" + return json.dumps(obj, ensure_ascii=False) + + +def _splice_object(s, span, old_obj, new_obj, fmt): + """Text of new_obj made from the old object's text at span, when both + are objects with the same keys in the same order: every member keeps + its text and only changed values are replaced (rendered on one line + when scalar, else with fmt at that member's indentation). None when + the shapes differ.""" + if not (isinstance(old_obj, dict) and isinstance(new_obj, dict) + and list(old_obj) == list(new_obj)): + return None + dec = json.JSONDecoder() + start, end = span + i = _WS.match(s, start + 1).end() + out, pos = [], start + try: + while s[i:i + 1] == '"': + key, k_end = dec.raw_decode(s, i) + j = _WS.match(s, k_end).end() + if s[j:j + 1] != ":": + return None + v_start = _WS.match(s, j + 1).end() + _val, v_end = dec.raw_decode(s, v_start) + if _key(old_obj.get(key)) != _key(new_obj.get(key)): + line_start = s.rfind("\n", 0, i) + 1 + indent = re.match(r"[ \t]*", s[line_start:i]).group(0) + text = json.dumps(new_obj[key], indent=fmt["indent"], + ensure_ascii=fmt["ensure_ascii"]) + out.append(s[pos:v_start]) + out.append(text.replace("\n", fmt["newline"] + indent)) + pos = v_end + j = _WS.match(s, v_end).end() + if s[j:j + 1] == ",": + i = _WS.match(s, j + 1).end() + else: + break + except ValueError: + return None + out.append(s[pos:end]) + return "".join(out) + + +def _splice_list(old, data, fmt): + """New text for a top-level JSON list that keeps the exact original + text of every entry that did not change (and the separators between + neighbours that stay neighbours); new or edited entries are rendered + with fmt at the first entry's indentation. None when old is not a + plain JSON list or the result would not parse back to data.""" + if not isinstance(data, list) or not data: + return None + s = old + dec = json.JSONDecoder() + i = _WS.match(s, 1 if s.startswith(_BOM) else 0).end() + if s[i:i + 1] != "[": + return None + head_end = i + 1 + i = _WS.match(s, head_end).end() + if s[i:i + 1] == "]": + return None + spans, items = [], [] + try: + while True: + obj, end = dec.raw_decode(s, i) + spans.append((i, end)) + items.append(obj) + j = _WS.match(s, end).end() + if s[j:j + 1] == ",": + i = _WS.match(s, j + 1).end() + continue + if s[j:j + 1] == "]": + close = j + break + return None + except ValueError: + return None + lead = s[head_end:spans[0][0]] + nl = fmt["newline"] + if "\n" in lead: + item_indent = lead.rsplit("\n", 1)[-1] + std_sep = "," + nl + item_indent + else: + item_indent, std_sep = "", ", " + + sm = difflib.SequenceMatcher(None, [_key(o) for o in items], [_key(o) for o in data], + autojunk=False) + # src: old entry identical to new entry n; origin: the old entry new + # entry n stands in for (identical, or edited in a one-for-one replace). + src = [None] * len(data) + origin = [None] * len(data) + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag == "equal" or (tag == "replace" and i2 - i1 == j2 - j1): + for k in range(j2 - j1): + origin[j1 + k] = i1 + k + if tag == "equal": + src[j1 + k] = i1 + k + + def fresh(obj): + text = json.dumps(obj, indent=fmt["indent"], ensure_ascii=fmt["ensure_ascii"]) + return text.replace("\n", nl + item_indent) + + def edited(n, obj): + # An entry that is one of the old ones with values changed (same + # keys, same order) keeps its own layout: only the values move. + k = origin[n] + if src[n] is not None: + return s[spans[k][0]:spans[k][1]] + if k is not None: + text = _splice_object(s, spans[k], items[k], obj, fmt) + if text is not None: + return text + return fresh(obj) + + out = [s[:head_end], lead] + for n, obj in enumerate(data): + if n: + a, b = origin[n - 1], origin[n] + out.append(s[spans[a][1]:spans[b][0]] if a is not None and b == a + 1 else std_sep) + out.append(edited(n, obj)) + out.append(s[spans[-1][1]:close]) + out.append(s[close:]) + text = "".join(out) + try: + if _key(json.loads(text.lstrip(_BOM))) != _key(data): + return None + except ValueError: + return None + return text + + +def plan_json(path, data, label=None): + """plan_text for a JSON document, rendered in the file's own format. + A file that does not round-trip keeps its unchanged entries verbatim + (_splice_list); "reformat" is set only when that is not possible.""" + old = read_text(path) + fmt = detect_json_format(old) + if old: + try: + same = render_json(json.loads(old.lstrip(_BOM)), fmt) == old + except ValueError: + same = False + if not same: + spliced = _splice_list(old, data, fmt) + if spliced is not None: + return plan_text(path, spliced, label) + plan = plan_text(path, render_json(data, fmt), label) + plan["reformat"] = True + return plan + return plan_text(path, render_json(data, fmt), label) + + +def public(plan): + """The part of a plan a page needs (no full file texts).""" + return {k: plan[k] for k in ("name", "changed", "diff", "sha_before", "existed", "reformat")} + + +# --- write --------------------------------------------------------------------- + +def _stamp(): + return datetime.now().strftime("%Y%m%d-%H%M%S") + + +def _match_owner(path, ref_stat): + if ref_stat is None: + return + try: + os.chown(path, ref_stat.st_uid, ref_stat.st_gid) + except (PermissionError, OSError): + pass # not root (local tests): the file stays ours + + +def _ref_stat(path): + """Stat to copy owner/mode from: the file, else its directory.""" + for p in (path, os.path.dirname(path)): + try: + return os.stat(p) + except OSError: + continue + return None + + +def atomic_write(path, text): + """Write text (see encode) or bytes via a temp file in the same + directory, keeping the old file's owner and mode. A symlinked control + file is written through to its target (os.replace on the link itself + would turn it into a separate copy).""" + path = os.path.realpath(path) + folder = os.path.dirname(path) + os.makedirs(folder, exist_ok=True) + existed = os.path.isfile(path) + ref = _ref_stat(path) + data = text if isinstance(text, bytes) else encode(text) + fd, tmp = tempfile.mkstemp(dir=folder, prefix=".webui-") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + fh.flush() + os.fsync(fh.fileno()) + if existed: + shutil.copymode(path, tmp) + else: + os.chmod(tmp, 0o664) + _match_owner(tmp, ref) + os.replace(tmp, path) + except BaseException: + if os.path.exists(tmp): + os.remove(tmp) + raise + + +def backup(path, tag=BACKUP_TAG, stamp=None): + """Copy path to .bak-- (owner kept). None if missing.""" + if not os.path.isfile(path): + return None + dst = f"{path}.bak-{stamp or _stamp()}-{tag}" + n = 1 + while os.path.exists(dst): # two saves in one second + n += 1 + dst = f"{path}.bak-{stamp or _stamp()}-{tag}{n}" + shutil.copy2(path, dst) + _match_owner(dst, _ref_stat(path)) + return dst + + +# --- undo records ------------------------------------------------------------------ + +def _undo_path(image_type, scope): + safe = re.sub(r"[^A-Za-z0-9._-]", "_", f"{image_type}--{scope}") + return os.path.join(uc.state_dir("webui-undo"), safe + ".json") + + +@contextlib.contextmanager +def _locked(): + """One lock for every commit/undo (saves are rare; threads and worker + processes both honour it, flock is per open file).""" + lock = os.path.join(uc.state_dir("webui-undo"), ".lock") + with open(lock, "a") as fh: + fcntl.flock(fh, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(fh, fcntl.LOCK_UN) + + +def _restore(path, bak): + """Put a file back as its backup holds it, or remove it (bak None).""" + if bak: + with open(bak, "rb") as fh: + atomic_write(path, fh.read()) + elif os.path.lexists(path): + os.remove(path) + + +def last_save(image_type, scope): + """The undo point for (image_type, scope): {"ts", "action", "files": + [{"path", "name", "backup", "existed", "sha_after"}]} or None.""" + return uc.read_json(_undo_path(image_type, scope), None) + + +def commit(image_type, scope, plans, action="", expect=None): + """Back up and write every changed plan; record the undo point. + Returns the list of changed plans (empty when nothing changed, and then + the previous undo point is kept). Raises Conflict when expect is given + and a file no longer has the previewed hash. When a write fails part + way, the files already written are put back before the error is raised.""" + expect = expect or {} + with _locked(): + changed = [p for p in plans if p["changed"]] + stale = [p["name"] for p in plans + if p["name"] in expect and expect[p["name"]] != file_sha256(p["path"])] + if stale: + raise Conflict("changed on disk since the preview: " + ", ".join(stale) + + ". Reload the page and review again.") + if not changed: + return [] + stamp = _stamp() + files = [] + try: + for p in changed: + bak = backup(p["path"], stamp=stamp) + files.append({"path": p["path"], "name": p["name"], "backup": bak, + "existed": bak is not None, "sha_after": sha256_text(p["new"])}) + atomic_write(p["path"], p["new"]) + uc.write_json(_undo_path(image_type, scope), + {"ts": uc.now(), "action": action, "files": files}) + except BaseException: + for f in reversed(files): + try: + _restore(f["path"], f["backup"]) + except OSError: + pass # the .bak file is still next to it + raise + return changed + + +def undo(image_type, scope): + """Restore the backups of the last commit for (image_type, scope). + The current files are themselves backed up (tag webui-undo) first, so + an undo can be undone by hand. Returns the record that was undone.""" + with _locked(): + rec = last_save(image_type, scope) + if not rec: + raise Conflict("nothing to undo") + moved = [f["name"] for f in rec["files"] + if file_sha256(f["path"]) != f["sha_after"]] + if moved: + raise Conflict("changed since that save, not undone: " + ", ".join(moved) + + ". The backups are still next to the files.") + missing = [f["name"] for f in rec["files"] + if f["existed"] and not os.path.isfile(f["backup"] or "")] + if missing: + raise Conflict("backup missing for: " + ", ".join(missing)) + stamp = _stamp() + for f in rec["files"]: + backup(f["path"], tag=BACKUP_TAG + "-undo", stamp=stamp) + _restore(f["path"], f["backup"] if f["existed"] else None) + os.remove(_undo_path(image_type, scope)) + return rec diff --git a/webapp/services/dell_updates.py b/webapp/services/dell_updates.py new file mode 100644 index 0000000..cc300b9 --- /dev/null +++ b/webapp/services/dell_updates.py @@ -0,0 +1,1422 @@ +"""Keep the Dell Win11 driver packs and the WinPE BIOS files current. + +What it manages: + * driver packs: every Win11 entry in the three images' HardwareDriver.json + (gea-standard, gea-engineer, gea-shopfloor; standard and engineer are + byte-identical and share the zips under _shared/Out-of-box Drivers); + * BIOS: the WinPE manifest models.txt in BIOS_DIR + (`ModelSubstring|BIOSFile|Version`, first substring match wins in + check-bios.cmd, so lines are never reordered). + +Where the answers come from (both cabs are plain XML inside, read with 7z): + * https://downloads.dell.com/catalog/DriverPackCatalog.cab: one + DriverPackage per pack, with OS codes, supported models and hashes; + * https://downloads.dell.com/catalog/CatalogIndexPC.cab: one GroupManifest + per system ID pointing at a small per-model cab whose BIOS + SoftwareComponent is the latest BIOS. Per-model cabs are cached and + checked against the sha256 the index gives. (CatalogPC.cab is stale and + is not used.) + +The flow, each step a function here and a button on /updates/dell: + 1. check() background job: fetch catalogs, write scan.json; + 2. download() background job: fetch the selected packs/BIOS at the speed + cap, verify Dell's sha256 (md5 fallback), turn each driver + exe into the deflate zip PESetup expects, stage it; + 3. preview()/apply(): place staged files next to the files they replace + (rename; staging and share are one filesystem), rewrite all + three maps and models.txt with backups, keep history; + 4. rollback(): put the maps / models.txt back from those backups. +Old zips and exes are never deleted. + +Run as a background job: python3 -m services.dell_updates check|download|apply +""" +from __future__ import annotations + +import fcntl +import glob +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from datetime import datetime +from pathlib import PurePosixPath + +import config +from services import updates_common as uc + +NS = "{openmanage/cm/dm}" +DELL = "https://downloads.dell.com/" +CATALOGS = { + "drivers": "catalog/DriverPackCatalog.cab", + "index": "catalog/CatalogIndexPC.cab", +} +# Images whose driver maps are scanned and rewritten (all deploy Win11). +IMAGES = ("gea-standard", "gea-engineer", "gea-shopfloor") +# PESetup osId values that mean Windows 11 (18 is Windows 10). +WIN11_OS_IDS = {"19", "20", "21"} +# Background job unit names (systemd-run --unit). +UNITS = {"check": "pxe-dell-check", "download": "pxe-dell-download"} +# Index brands that are never on this site's floor; skipping them keeps +# substrings like "7090" from matching consumer models. +_SKIP_BRANDS = re.compile(r"inspiron|xps|vostro|alienware|g series|chromebook|wyse|edge|poweredge", re.I) +# Seen models that have no Dell packs by design. +_SKIP_MODELS = ("virtual", "vmware", "qemu", "hyper-v", "kvm") + + +# --- paths and small helpers -------------------------------------------------- + +def _dir(*parts): + return uc.state_dir("dell", *parts) + + +def _path(name): + return os.path.join(_dir(), name) + + +def _stamp(): + return datetime.now().strftime("%Y%m%d-%H%M%S") + + +def _sha256(path): + return uc.file_hash(path, "sha256").upper() + + +def _alnum(s): + return re.sub(r"[^a-z0-9]", "", str(s or "").lower()) + + +def _nice(): + """Prefix for CPU/disk heavy commands so imaging traffic keeps priority.""" + cmd = [] + if shutil.which("ionice"): + cmd += ["ionice", "-c", "3"] + if shutil.which("nice"): + cmd += ["nice", "-n", "15"] + return cmd + + +def _cab_root(cab): + """Parse the XML inside a Dell cab (UTF-8 or UTF-16, 7z extracts both).""" + r = subprocess.run(["7z", "e", "-so", cab], capture_output=True) + if r.returncode or not r.stdout: + raise RuntimeError(f"cannot extract {os.path.basename(cab)}: " + f"{r.stderr.decode(errors='replace').strip()[:200]}") + return ET.fromstring(r.stdout) + + +def _hashes(elem): + """{"SHA256": hex, "MD5": hex, ...} from an element's own Cryptography.""" + out = {} + crypt = elem.find(NS + "Cryptography") if elem is not None else None + if crypt is not None: + for h in crypt.findall(NS + "Hash"): + if h.text: + out[(h.get("algorithm") or "").upper()] = h.text.strip() + return out + + +def _int(v): + try: + return int(v) + except (TypeError, ValueError): + return 0 + + +# check-bios.cmd expands the BIOSFile and Version fields unquoted in cmd +# (`set "BIOSFILE=%%B"`, `find /I "%TARGETVER%"`), so a name with & ^ % ! ( ) +# or a space would break every WinPE boot. Every real Dell BIOS name fits. +_CMD_SAFE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + + +def bios_name_ok(name, version): + return bool(_CMD_SAFE.fullmatch(str(name or "")) and _CMD_SAFE.fullmatch(str(version or ""))) + + +def file_name_ok(name): + """A plain file name from Dell's catalog: no path, no leading dot. Some + Win11 CAB packs have spaces ('7390 2in1-win11-A00-4PY3P.CAB'), which is + fine for a staging file run through argv lists.""" + name = str(name or "") + return (bool(name) and name.isascii() and name.isprintable() and not name.startswith(".") + and "/" not in name and "\\" not in name) + + +# --- name and version rules ----------------------------------------------------- + +def norm(s): + """Model name for matching: lower case, no 'Dell', AIO spelled one way.""" + s = str(s or "").lower().replace("all-in-one", "aio").replace("all in one", "aio") + s = s.replace("optiplex", "optiplex ") + s = re.sub(r"\bdell\b", "", s) + s = re.sub(r"[^a-z0-9]+", " ", s) + return " ".join(s.split()) + + +def aversion(name): + """Dell A-number from a pack or zip name: ..._a08.zip -> 8, None if absent.""" + m = re.search(r"(?i)(?:^|[_-])a(\d+)(?=[_.\-(]|$)", str(name or "")) + return int(m.group(1)) if m else None + + +def pack_code(file_name, release_id=""): + """Model code of a Dell pack: PB13250-HT8VV_Win11_1.0_A08.exe -> pb13250, + OptiPlex-3000-JN664_Win11_... -> optiplex3000, and the older CAB packs + '7390 2in1-win11-A00-4PY3P.CAB' -> 73902in1.""" + base = os.path.basename(file_name) + m = re.match(r"(?i)(.+?)[_-]win1[01]", base) + stem = m.group(1) if m else base.split("_")[0] + if release_id and stem.lower().endswith("-" + release_id.lower()): + stem = stem[: -len(release_id) - 1] + elif not release_id: + stem = re.sub(r"-[A-Za-z0-9]{5}$", "", stem) + return _alnum(stem) + + +def zip_name(file_name, release_id, dell_version): + """win11__.zip, the name the driver zip is staged under.""" + return f"win11_{pack_code(file_name, release_id)}_{_alnum(dell_version)}.zip" + + +def vtuple(v): + """Comparable BIOS version: '1.45.0' -> (1, 45, 0), '02.12.00' -> (2, 12, 0), + 'A22' -> (22,).""" + v = str(v or "").strip() + m = re.fullmatch(r"(?i)a(\d+)", v) + if m: + return (int(m.group(1)),) + return tuple(int(x) for x in re.findall(r"\d+", v)) + + +def norm_version(v): + """Version as models.txt writes it: dotted numbers without leading zeros.""" + v = str(v or "").strip() + if re.fullmatch(r"[\d._]+", v): + return ".".join(str(int(x)) for x in re.split(r"[._]", v) if x) + return v + + +def bios_stem(file_name): + """BIOS file family: the name with its version taken out, lower case. + Latitude_5430_7330_Rugged_1.45.0.exe -> latitude_5430_7330_rugged_#.exe""" + base = os.path.basename(str(file_name or "")).lower() + hits = list(re.finditer(r"\d+(?:[._]\d+){1,3}", base)) + if hits: + m = hits[-1] + return base[: m.start()] + "#" + base[m.end():] + return re.sub(r"a\d+(?=\.exe$)", "#", base) + + +# --- reading the share ---------------------------------------------------------- + +def map_path(image): + return os.path.join(config.SAMBA_SHARE, image, "Deploy", "Control", "HardwareDriver.json") + + +def read_maps(errors=None): + """{image: {"path", "raw", "entries"}} for every image map present. + A map that exists but cannot be read or parsed is left out and, when + errors is a list, reported there: apply must not quietly update two of + three images.""" + out = {} + for image in IMAGES: + path = map_path(image) + if not os.path.lexists(path): + continue # image not on this server + try: + with open(path, "rb") as f: + raw = f.read() + entries = json.loads(raw.decode("utf-8-sig")) + except (OSError, ValueError) as ex: + if errors is not None: + errors.append(f"{image} HardwareDriver.json cannot be read ({str(ex)[:120]}); " + "that image is not updated") + continue + if isinstance(entries, list): + out[image] = {"path": path, "raw": raw, "entries": entries} + elif errors is not None: + errors.append(f"{image} HardwareDriver.json is not a list; that image is not updated") + return out + + +def is_win11(entry): + ids = set(str(entry.get("osId") or "").replace(" ", "").split(",")) + return bool(ids & WIN11_OS_IDS) + + +def tokens(entry): + return [t.strip() for t in str(entry.get("modelswminame") or "").split(",") if t.strip()] + + +def entry_file(entry): + return str(entry.get("fileName") or entry.get("FileName") or "") + + +def entry_dir(entry): + """destinationDir as a path relative to SHARED_DIR ("Out-of-box Drivers/...").""" + d = str(entry.get("destinationDir") or entry.get("DestinationDir") or "") + d = re.sub(r"(?i)^\*destinationdir\*\\?", "", d).replace("\\", "/").strip("/") + if d.lower().startswith("deploy/"): + d = d[len("deploy/"):] + return d + + +def models_txt_path(): + return os.path.join(config.BIOS_DIR, "models.txt") + + +def read_models_txt(): + """[(index, raw line without CR/LF, substring, file, version)] for each + manifest line, plus the raw text.""" + try: + with open(models_txt_path(), "rb") as f: + text = f.read().decode("ascii", errors="replace") + except OSError: + return [], "" + rows = [] + for i, line in enumerate(text.split("\n")): + line = line.rstrip("\r") + if not line.strip() or line.lstrip().startswith("#") or line.count("|") < 2: + continue + sub, bfile, ver = (x.strip() for x in line.split("|")[:3]) + if sub: + rows.append((i, line, sub, bfile, ver)) + return rows, text + + +def resolve_dir_ci(root, relpath, create=False): + """root/relpath matching each existing component case-insensitively (the + share has Dell_11/Optiplex and an OptiPlex symlink to it). With create, + missing components are made (owned like their parent). None when absent.""" + cur = root + for part in PurePosixPath(relpath).parts: + try: + names = os.listdir(cur) + except OSError: + names = [] + hit = next((n for n in names if n == part), None) or \ + next((n for n in names if n.lower() == part.lower()), None) + if hit is None: + if not create: + return None + new = os.path.join(cur, part) + os.makedirs(new, exist_ok=True) + _chown_like(new, cur) + cur = new + else: + cur = os.path.join(cur, hit) + return cur if os.path.isdir(cur) else None + + +def _chown_like(path, ref): + try: + st = os.stat(ref) + os.chown(path, st.st_uid, st.st_gid) + except OSError: + pass # not root (local tests): leave as is + + +# --- Dell catalogs -------------------------------------------------------------- + +def _catalog_file(kind): + return os.path.join(_dir("catalog"), os.path.basename(CATALOGS[kind])) + + +def fetch_catalogs(): + """Download both catalog cabs into the cache (they carry no hash of their + own; the per-model cabs they list do).""" + for kind, rel in CATALOGS.items(): + dest = _catalog_file(kind) + tmp = dest + ".part" + if os.path.exists(tmp): + os.remove(tmp) + uc.download(DELL + rel, tmp, timeout=300) + _cab_root(tmp) # refuse a cab that does not parse before replacing + os.replace(tmp, dest) + + +def driver_catalog(root): + """Win11 driver packs: [{models, sids, ver, path, name, date, size, sha256, + md5, release_id}], plus the catalog's own date.""" + packs = [] + for p in root.findall(NS + "DriverPackage"): + oss = {o.get("osCode") for o in p.iter(NS + "OperatingSystem")} + if "Windows11" not in oss: + continue + h = _hashes(p) + path = p.get("path") or "" + packs.append({ + "models": [m.get("name") or "" for m in p.iter(NS + "Model")], + "sids": sorted({(m.get("systemID") or "").upper() for m in p.iter(NS + "Model")} - {""}), + "ver": p.get("dellVersion") or "", + "path": path, + "name": path.rsplit("/", 1)[-1], + "date": (p.get("dateTime") or "")[:10], + "size": _int(p.get("size")), + "sha256": h.get("SHA256", ""), + "md5": h.get("MD5", "") or (p.get("hashMD5") or ""), + "release_id": p.get("releaseID") or "", + }) + return packs + + +def catalog_index(root): + """Per-model BIOS catalogs: [{brand, name, full, sid, path, sha256, size}].""" + idx = [] + for g in root.findall(NS + "GroupManifest"): + b = g.find(f".//{NS}Brand/{NS}Display") + brand = (b.text or "").strip() if b is not None else "" + if _SKIP_BRANDS.search(brand): + continue + mi = g.find(NS + "ManifestInformation") + if mi is None or not mi.get("path"): + continue + h = _hashes(mi) + for m in g.iter(NS + "Model"): + d = m.find(NS + "Display") + name = (d.text or "").strip() if d is not None else "" + idx.append({"brand": brand, "name": name, "full": norm(brand + " " + name), + "sid": (m.get("systemID") or "").upper(), "path": mi.get("path"), + "sha256": h.get("SHA256", ""), "size": _int(mi.get("size"))}) + return idx + + +class ModelCabs: + """Latest BIOS per per-model cab, downloaded once and hash checked.""" + + def __init__(self, fetch=True): + self.fetch = fetch + self.cache = {} + self.errors = {} + + def latest_bios(self, ix): + path = ix["path"] + if path in self.cache: + return self.cache[path] + best = None + try: + local = os.path.join(_dir("catalog", "models"), path.replace("/", "_")) + fresh = os.path.exists(local) and ( + not ix["sha256"] or uc.file_hash(local).lower() == ix["sha256"].lower()) + if not fresh: + if not self.fetch: + raise RuntimeError("per-model catalog not cached") + if os.path.exists(local): + os.remove(local) + uc.download(DELL + path, local, expect=ix["sha256"] or None, timeout=120) + root = _cab_root(local) + for c in root.iter(NS + "SoftwareComponent"): + ct = c.find(NS + "ComponentType") + if ct is None or ct.get("value") != "BIOS": + continue + cpath = c.get("path") or "" + version = c.get("vendorVersion") or c.get("dellVersion") or "" + h = _hashes(c) + cand = { + "file": cpath.rsplit("/", 1)[-1], "path": cpath, "url": DELL + cpath, + "version": norm_version(version), "date": c.get("releaseDate") or "", + "size": _int(c.get("size")), "sha256": h.get("SHA256", "").upper(), + "md5": h.get("MD5", "").upper(), + } + if best is None or vtuple(cand["version"]) > vtuple(best["version"]): + best = cand + except Exception as ex: # one bad cab must not sink the whole scan + self.errors[path] = str(ex)[:200] + self.cache[path] = best + return best + + +# --- matching ------------------------------------------------------------------- + +def _rank(t_words, c_words): + """How well a catalog model name fits a modelswminame token, or None. + The model numbers (words with a digit) of the token must all be in the + catalog name; then an exact word set beats one name containing the + other ('OptiPlex Tower Plus 7020' vs 'OptiPlex 7020 Tower'), which beats + a bare number match. Fewer extra words is better.""" + t, c = set(t_words), set(c_words) + digits = {w for w in t if any(ch.isdigit() for ch in w)} + if not digits or not digits <= c: + return None + if t == c: + return (3, 0) + if t <= c or c <= t: + return (2, -len(t ^ c)) + if (t - digits) & c: + return (1, -len(t ^ c)) + return None + + +def _pack_hits(token, packs): + """Win11 packs whose model names best fit one modelswminame token.""" + nn = norm(token).split() + if not nn: + return [] + scored = [] + for c in packs: + ranks = [r for r in (_rank(nn, norm(x).split()) for x in c["models"]) if r] + if ranks: + scored.append((max(ranks), c)) + if not scored: + return [] + top = max(r for r, _ in scored) + return [c for r, c in scored if r == top] + + +def _best(packs): + return max(packs, key=lambda c: (aversion("_" + c["ver"]) or 0, c["date"])) if packs else None + + +def _pack_view(c): + return { + "key": pack_code(c["name"], c["release_id"]) + "-" + _alnum(c["ver"]), + "path": c["path"], "url": DELL + c["path"], "name": c["name"], "ver": c["ver"], + "date": c["date"], "size": c["size"], "sha256": c["sha256"].upper(), + "md5": c["md5"].upper(), "release_id": c["release_id"], + "zip": zip_name(c["name"], c["release_id"], c["ver"]), + "code": pack_code(c["name"], c["release_id"]), + "models": sorted(set(c["models"])), + } + + +def scan_drivers(maps, packs): + """One row per distinct current file in the Win11 entries of all maps.""" + groups = {} + for image, m in maps.items(): + for e in m["entries"]: + if not isinstance(e, dict) or not is_win11(e): + continue + fn = entry_file(e) + if not fn: + continue + g = groups.setdefault(fn, {"tokens": [], "images": [], "dirs": []}) + for t in tokens(e): + if t not in g["tokens"]: + g["tokens"].append(t) + if image not in g["images"]: + g["images"].append(image) + d = entry_dir(e) + if d not in g["dirs"]: + g["dirs"].append(d) + rows = [] + for fn, g in groups.items(): + per_token, codes = {}, set() + for t in g["tokens"]: + hits = _pack_hits(t, packs) + codes |= {pack_code(c["name"], c["release_id"]) for c in hits} + b = _best(hits) + if b: + per_token[t] = b + best = _best(list(per_token.values())) + ours = aversion(fn) + warnings, notes = [], [] + if not best: + status = "no-match" + elif ours is None: + status = "unknown-version" + elif (aversion("_" + best["ver"]) or 0) > ours: + status = "update" + else: + status = "current" + pack = _pack_view(best) if best else None + if pack: + if len(codes) > 1: + warnings.append( + f"The model names in this entry match {len(codes)} different Dell packs " + f"({', '.join(sorted(codes))}); one zip cannot cover them all, so check " + "which one fits or split the entry by hand.") + if fn.lower().startswith("win10") or "_win10_" in fn.lower(): + warnings.append("The current file is a Win10 pack, so its A-number is not comparable.") + elif pack["code"] and pack["code"] not in _alnum(fn): + warnings.append( + f"The current file looks like a GE/MCL family pack (its name lacks " + f"'{pack['code']}'), so its A-number may not be comparable. Decide by hand.") + if "nvidia" in fn.lower(): + notes.append("The current pack carries NVIDIA drivers; the update is skipped " + "if the Dell pack has no NVIDIA INFs.") + rows.append({ + "id": "d-" + hashlib.sha1(fn.encode("utf-8")).hexdigest()[:10], + "current": fn, "cur_ver": f"A{ours:02d}" if ours is not None else "", + "tokens": g["tokens"], "images": g["images"], "dirs": g["dirs"], + "status": status, "pack": pack, "warnings": warnings, "notes": notes, + "default": status == "update" and not warnings, + }) + order = {"update": 0, "unknown-version": 1, "no-match": 2, "current": 3} + rows.sort(key=lambda r: (order.get(r["status"], 9), ", ".join(r["tokens"]).lower())) + return rows + + +def _variants(ix): + """The model names one index entry stands for, lower case, as check-bios + would see them: 'Dell Pro 7 13 P713260/Dell Pro 7 13 2-in-1 P703260' is + two machines sharing one per-model catalog.""" + out = [] + for v in ix["name"].split("/"): + v = " ".join(v.split()).lower() + if v: + out.append(v) + out.append((ix["brand"].lower() + " " + v).strip()) + return out + + +def _line_owner(lines, texts): + """Index into lines of the first manifest line check-bios.cmd would pick + for a model called any of texts (literal substring, case-insensitive).""" + for n, (_, _, sub, _, _) in enumerate(lines): + s = sub.lower() + if any(s in t for t in texts): + return n + return None + + +def _merge_by_family(cands): + """One candidate per file family (stem), the highest version, models merged.""" + fam = {} + for c in cands: + k = bios_stem(c["file"]) + cur = fam.get(k) + if cur is None or vtuple(c["version"]) > vtuple(cur["version"]): + merged = dict(c, models=sorted(set(c["models"]) | set(cur["models"] if cur else []))) + fam[k] = merged + else: + cur["models"] = sorted(set(cur["models"]) | set(c["models"])) + return list(fam.values()) + + +def scan_bios(lines, index, cabs): + """One row per models.txt line. Each Dell model is credited to the first + line that would catch it, as check-bios.cmd does, so '7090' does not + report the models '7090 Ultra' (an earlier line) already owns. A line no + Dell name contains literally (Dell lists 'OptiPlex Micro 7010' as just + '7010') falls back to a model-number match that prefers our own file + family.""" + owned = {} + for ix in index: + for v in ix["name"].split("/") or [ix["name"]]: + n = _line_owner(lines, [t for t in _variants(dict(ix, name=v))]) + if n is not None and ix not in owned.setdefault(n, []): + owned[n].append(ix) + rows = [] + for n, (i, raw, sub, bfile, ver) in enumerate(lines): + loose = n not in owned + models = owned.get(n, []) + if loose: + digits = {w for w in norm(sub).split() if any(ch.isdigit() for ch in w)} + models = [ix for ix in index if digits and digits <= set(norm(ix["brand"] + " " + ix["name"]).split())] + cands = [] + for ix in models: + b = cabs.latest_bios(ix) + if b: + cands.append(dict(b, models=[(ix["brand"] + " " + ix["name"]).strip()])) + ours_stem = bios_stem(bfile) + fam = _merge_by_family(cands) + warnings, notes = [], [] + if loose and fam: + mine = [c for c in fam if bios_stem(c["file"]) == ours_stem] + notes.append("No Dell model name contains this line; matched by model number only.") + if mine: + fam = mine + fam.sort(key=lambda c: (bios_stem(c["file"]) != ours_stem, tuple(-x for x in vtuple(c["version"])))) + if not fam: + errs = [cabs.errors[ix["path"]] for ix in models if ix["path"] in cabs.errors] + status = "error" if errs else "no-match" + if errs: + warnings.append("Per-model catalog failed: " + errs[0]) + elif len(fam) > 1: + status = "ambiguous" + warnings.append(f"{len(fam)} different BIOS file families match this line; " + "check-bios.cmd would flash the same file on all of them. Pick one.") + else: + c = fam[0] + if bios_stem(c["file"]) != ours_stem: + status = "different-file-family" + warnings.append("Dell's latest BIOS for these models is a different file family.") + elif vtuple(c["version"]) > vtuple(ver): + status = "update" + else: + status = "current" + rows.append({ + "id": f"b-{i}", "line_no": i, "line": raw, "sub": sub, "current": bfile, + "cur_ver": ver, "status": status, "candidates": fam, "pick": 0, + "warnings": warnings, "notes": notes, "default": status == "update", + }) + order = {"update": 0, "different-file-family": 1, "ambiguous": 2, "error": 3, + "no-match": 4, "current": 5} + rows.sort(key=lambda r: (order.get(r["status"], 9), r["line_no"])) + return rows + + +def _entry_matches(entry, model): + """PESetup's matcher (see services/hardware_coverage.py).""" + try: + from services import hardware_coverage + return hardware_coverage._entry_matches(entry, model) + except (ImportError, AttributeError): + return any(t.lower() in model.lower() + for t in str(entry.get("modelswminame") or "").split(",")) + + +def scan_missing(maps, lines, packs, index, cabs): + """Models WinPE has seen with no Win11 driver entry or no BIOS line, and + the Dell pack/BIOS that would cover them. Informational only.""" + state = uc.read_json(config.HARDWARE_STATE, {}) or {} + out = [] + for model, rec in sorted((state.get("models") or {}).items()): + if any(s in model.lower() for s in _SKIP_MODELS): + continue + no_drv = [img for img, m in maps.items() + if not any(isinstance(e, dict) and is_win11(e) and _entry_matches(e, model) + for e in m["entries"])] + no_bios = not any(sub.lower() in model.lower() for _, _, sub, _, _ in lines) + if not no_drv and not no_bios: + continue + skus = [s.upper() for s in rec.get("skus") or [] if s] + pack = bios = None + if no_drv: + hits = [c for c in packs if set(c["sids"]) & set(skus)] or _pack_hits(model, packs) + b = _best(hits) + pack = _pack_view(b) if b else None + if no_bios: + want = norm(model) + ixs = [ix for ix in index if ix["sid"] in skus] or \ + [ix for ix in index if want in {norm(v) for v in _variants(ix)}] + for ix in ixs: + bios = cabs.latest_bios(ix) + if bios: + break + out.append({"model": model, "skus": skus, "no_drivers": no_drv, + "no_bios": no_bios, "pack": pack, "bios": bios, + "last_seen": rec.get("last_seen", "")}) + return out + + +# --- 1. check ------------------------------------------------------------------- + +def check_state(): + return uc.read_json(_path("check.json"), {}) or {} + + +def _check_progress(**kw): + state = check_state() + state.update(kw) + uc.write_json(_path("check.json"), state) + + +def last_scan(): + return uc.read_json(_path("scan.json"), None) + + +def check(fetch=True): + """Fetch the catalogs (unless fetch is False: cached cabs only), compare + everything, write scan.json and return it.""" + uc.write_json(_path("check.json"), {"state": "running", "started": uc.now(), + "step": "Fetching Dell catalogs"}) + try: + if fetch: + fetch_catalogs() + _check_progress(step="Reading catalogs") + droot = _cab_root(_catalog_file("drivers")) + iroot = _cab_root(_catalog_file("index")) + packs = driver_catalog(droot) + index = catalog_index(iroot) + map_errors = [] + maps = read_maps(map_errors) + lines, _ = read_models_txt() + _check_progress(step="Comparing driver packs") + drivers = scan_drivers(maps, packs) + cabs = ModelCabs(fetch=fetch) + _check_progress(step="Reading per-model BIOS catalogs") + bios = scan_bios(lines, index, cabs) + missing = scan_missing(maps, lines, packs, index, cabs) + result = { + "checked": uc.now(), + "catalogs": {"drivers": droot.get("dateTime") or droot.get("version") or "", + "index": iroot.get("creationDateTime") or iroot.get("version") or ""}, + "images": sorted(maps), + "drivers": drivers, "bios": bios, "missing": missing, + "cab_errors": cabs.errors, "map_errors": map_errors, + } + uc.write_json(_path("scan.json"), result) + _check_progress(state="done", finished=uc.now(), step="") + return result + except Exception as ex: + _check_progress(state="error", finished=uc.now(), error=str(ex)[:400]) + raise + + +# --- 2. download ---------------------------------------------------------------- + +def staged_state(): + return uc.read_json(_path("staged.json"), {}) or {} + + +def _save_staged(state): + uc.write_json(_path("staged.json"), state) + + +def _set_item(key, **kw): + state = staged_state() + state.setdefault("items", {}).setdefault(key, {}).update(kw) + _save_staged(state) + + +def select(driver_ids, bios_picks): + """Turn checked rows of the last scan into a download list. + driver_ids: scan row ids; bios_picks: {row id: candidate index}. + Rows sharing one Dell file become one item. Returns the item count.""" + scan = last_scan() + if not scan: + raise ValueError("no scan yet: run a check first") + items = {} + for row in scan["drivers"]: + if row["id"] not in driver_ids or not row.get("pack"): + continue + p = row["pack"] + if not file_name_ok(p["name"]) or not _CMD_SAFE.fullmatch(p["zip"]): + raise ValueError(f"refusing Dell pack with an unsafe file name: {p['name']!r}") + it = items.setdefault(p["key"], { + "key": p["key"], "kind": "driver", "url": p["url"], "path": p["path"], + "name": p["name"], "size": p["size"], "sha256": p["sha256"], "md5": p["md5"], + "ver": p["ver"], "date": p["date"], "zip": p["zip"], "replaces": []}) + if row["current"] not in it["replaces"]: + it["replaces"].append(row["current"]) + for row in scan["bios"]: + if row["id"] not in bios_picks or not row["candidates"]: + continue + n = bios_picks[row["id"]] + if not 0 <= n < len(row["candidates"]): + n = 0 + c = row["candidates"][n] + if not bios_name_ok(c["file"], c["version"]): + raise ValueError(f"refusing BIOS {c['file']!r} version {c['version']!r}: " + "not safe in models.txt / check-bios.cmd") + key = "bios-" + _alnum(c["file"]) + it = items.setdefault(key, { + "key": key, "kind": "bios", "url": c["url"], "path": c["path"], + "name": c["file"], "size": c["size"], "sha256": c["sha256"], "md5": c["md5"], + "ver": c["version"], "date": c["date"], "lines": []}) + it["lines"].append({"line_no": row["line_no"], "line": row["line"], + "sub": row["sub"], "current": row["current"]}) + if not items: + raise ValueError("nothing selected") + uc.write_json(_path("selection.json"), {"created": uc.now(), "items": list(items.values())}) + return len(items) + + +def _free_bytes(path): + try: + return shutil.disk_usage(path).free + except OSError: + return 0 + + +def _is_nvidia_inf(path, rel): + """An NVIDIA INF: under a folder named NVIDIA..., an nv* INF whose text + names NVIDIA, or any INF whose Provider line does (INFs are ANSI or + UTF-16). A bare 'nv' prefix is not enough: nvme*.inf storage drivers + would make a pack without NVIDIA look like it has some, and the NVIDIA + rule would then replace a GE pack that carries them.""" + if any(part.lower().startswith("nvidia") for part in rel.split(os.sep)[:-1]): + return True + try: + with open(path, "rb") as f: + data = f.read(1 << 20) + except OSError: + return False + utf16 = data[:2] in (b"\xff\xfe", b"\xfe\xff") or data[1:200:2].count(0) > 50 + text = data.decode("utf-16" if utf16 else "latin-1", errors="replace").lower() + if os.path.basename(rel).lower().startswith("nv") and "nvidia" in text: + return True + return bool(re.search(r"^\s*provider\s*=[^\n]*nvidia", text, re.M)) + + +def _count_infs(folder): + infs = [p for p in glob.glob(os.path.join(folder, "**", "*"), recursive=True) + if p.lower().endswith(".inf")] + nv = [p for p in infs if _is_nvidia_inf(p, os.path.relpath(p, folder))] + return len(infs), len(nv) + + +def _stage_one(it): + key = it["key"] + if not file_name_ok(it["name"]) or not _CMD_SAFE.fullmatch(key): + raise RuntimeError(f"unsafe file name {it['name']!r}; refusing it") + folder = _dir("staged", key) + exe = os.path.join(folder, it["name"]) + algo, expect = ("sha256", it["sha256"]) if it.get("sha256") else ("md5", it.get("md5")) + if not expect: + raise RuntimeError("Dell lists no hash for this file; refusing it") + need = it["size"] * (5 if it["kind"] == "driver" else 1) + (1 << 30) + have = os.path.getsize(exe) if os.path.exists(exe) else 0 + if _free_bytes(folder) + have < need: + raise RuntimeError(f"not enough disk space in {folder} (need about {need // (1 << 30)} GB)") + if not (have == it["size"] and uc.file_hash(exe, algo).lower() == expect.lower()): + _set_item(key, status="downloading", partial=exe, error="") + uc.download(it["url"], exe, expect=expect, algorithm=algo) + _set_item(key, status="verified", partial="") + if it["kind"] == "bios": + _set_item(key, status="ready", file=exe, sha256_file=_sha256(exe), + size_file=os.path.getsize(exe), finished=uc.now()) + return + # Driver pack: 7z-extract the exe and deflate-zip its top folder(s). + work = os.path.join(folder, "x") + shutil.rmtree(work, ignore_errors=True) + os.makedirs(work) + try: + _set_item(key, status="extracting") + r = subprocess.run(_nice() + ["7z", "x", "-y", "-o" + work, exe], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + if r.returncode: + raise RuntimeError(f"7z failed ({r.returncode}): {r.stderr.strip()[:200]}") + tops = sorted(os.listdir(work)) + infs, nvidia = _count_infs(work) + if not tops or not infs: + raise RuntimeError("the pack extracted to no INF files") + _set_item(key, status="zipping", tops=tops, infs=infs, nvidia_infs=nvidia) + final = os.path.join(folder, it["zip"]) + tmp = os.path.join(folder, ".tmp-" + it["zip"]) + if os.path.exists(tmp): + os.remove(tmp) + r = subprocess.run(_nice() + ["zip", "-r", "-q", "-X", tmp] + tops, cwd=work, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + if r.returncode: + raise RuntimeError(f"zip failed ({r.returncode}): {r.stderr.strip()[:200]}") + os.replace(tmp, final) + finally: + shutil.rmtree(work, ignore_errors=True) + _set_item(key, status="ready", file=final, sha256_file=_sha256(final), + size_file=os.path.getsize(final), finished=uc.now()) + os.remove(exe) # the zip is what gets deployed; the exe can be fetched again + + +def download(): + """Background job: stage every item of selection.json. Resumable: ready + items are skipped, a partial exe is resumed.""" + sel = uc.read_json(_path("selection.json"), {}) or {} + state = staged_state() + state.update(state="running", started=uc.now(), finished="", error="") + items = state.setdefault("items", {}) + for it in sel.get("items", []): + cur = items.get(it["key"], {}) + if cur.get("status") == "ready" and os.path.exists(cur.get("file", "")): + # Already staged: just widen what it replaces. + for k in ("replaces", "lines"): + if k in it: + merged = cur.get(k, []) + for v in it[k]: + if v not in merged: + merged.append(v) + cur[k] = merged + continue + items[it["key"]] = dict(it, status="queued", error="", queued=uc.now()) + _save_staged(state) + failed = 0 + for it in sel.get("items", []): + if staged_state()["items"][it["key"]].get("status") == "ready": + continue + try: + _stage_one(it) + except Exception as ex: + failed += 1 + _set_item(it["key"], status="error", error=str(ex)[:300], partial="") + state = staged_state() + state.update(state="done", finished=uc.now(), + error=f"{failed} item(s) failed" if failed else "") + _save_staged(state) + _audit("DELL_DOWNLOAD", f"{len(sel.get('items', []))} item(s), {failed} failed") + + +def progress(): + """staged.json with live percent for items still downloading, and whether + the jobs are running.""" + state = staged_state() + running = uc.job_running(UNITS["download"]) + if state.get("state") == "running" and not running: + state["state"] = "interrupted" + for it in (state.get("items") or {}).values(): + pct = None + if it.get("status") == "downloading" and it.get("partial") and it.get("size"): + try: + pct = min(100.0, round(100.0 * os.path.getsize(it["partial"]) / it["size"], 1)) + except OSError: + pct = 0.0 + elif it.get("status") in ("verified", "extracting", "zipping", "ready", "applied"): + pct = 100.0 + it["percent"] = pct + if not running and it.get("status") in ("queued", "downloading", "verified", + "extracting", "zipping"): + it["status"] = "interrupted" + chk = check_state() + if chk.get("state") == "running" and not uc.job_running(UNITS["check"]): + chk["state"] = "interrupted" + return {"download": state, "check": chk, "download_running": running, + "check_running": uc.job_running(UNITS["check"])} + + +def cancel_download(): + uc.stop_job(UNITS["download"]) + state = staged_state() + for it in (state.get("items") or {}).values(): + if it.get("status") in ("queued", "downloading", "verified", "extracting", "zipping"): + it["status"] = "cancelled" + if state.get("state") == "running": + state["state"] = "cancelled" + state["finished"] = uc.now() + _save_staged(state) + + +def discard(keys): + """Drop staged items and their files (staging only; never the share).""" + state = staged_state() + items = state.get("items") or {} + gone = [] + for key in keys: + if key not in items or "/" in key or key.startswith("."): + continue + shutil.rmtree(os.path.join(_dir("staged"), key), ignore_errors=True) + items.pop(key) + gone.append(key) + _save_staged(state) + return gone + + +# --- 3. apply ------------------------------------------------------------------- + +def _ready(keys): + items = (staged_state().get("items") or {}) + return [items[k] for k in keys if k in items and items[k].get("status") == "ready" + and os.path.exists(items[k].get("file", ""))] + + +def _new_line(raw, bfile, ver): + parts = raw.split("|") + parts[1], parts[2] = bfile, ver + return "|".join(parts) + + +def preview(keys): + """What apply(keys) would change, without changing anything.""" + map_errors = [] + maps = read_maps(map_errors) + lines_by_no = {} + _, text = read_models_txt() + for n, line in enumerate(text.split("\n")): + lines_by_no[n] = line.rstrip("\r") + out = {"drivers": {img: [] for img in maps}, "bios": [], "skipped": [], "missing": [], + "errors": map_errors} + items = _ready(keys) + out["missing"] = [k for k in keys if k not in {i["key"] for i in items}] + for it in items: + if it["kind"] == "driver": + for cur in it.get("replaces", []): + if "nvidia" in cur.lower() and not it.get("nvidia_infs"): + out["skipped"].append(f"{cur}: has NVIDIA drivers, Dell pack {it['zip']} has none") + continue + hit = False + for img, m in maps.items(): + for e in m["entries"]: + if isinstance(e, dict) and entry_file(e) == cur: + hit = True + out["drivers"][img].append({ + "models": e.get("modelswminame", ""), "old": cur, + "new": it["zip"], "dir": entry_dir(e)}) + if not hit: + out["skipped"].append(f"{cur}: no map entry uses it any more") + else: + if not bios_name_ok(it["name"], it["ver"]): + out["errors"].append(f"{it['name']}: file name or version not safe for check-bios.cmd") + continue + for ln in it.get("lines", []): + n = _find_line(lines_by_no, ln) + if n is None: + out["skipped"].append(f"models.txt line '{ln['line']}' changed since the check") + continue + out["bios"].append({"old": lines_by_no[n], + "new": _new_line(lines_by_no[n], it["name"], it["ver"])}) + return out + + +def _find_line(lines_by_no, ln): + """The line's current index: where the scan saw it, or its unique copy.""" + if lines_by_no.get(ln["line_no"]) == ln["line"]: + return ln["line_no"] + hits = [n for n, t in lines_by_no.items() if t == ln["line"]] + return hits[0] if len(hits) == 1 else None + + +def _same_file(path, size, sha256): + return os.path.getsize(path) == size and _sha256(path) == sha256.upper() + + +def _place(src, dest_dir, name, size, sha256, move): + """Put src into dest_dir/name. Rename when move and on one filesystem, + else copy and verify. An existing file is accepted only if identical. + Returns the placed path.""" + dst = os.path.join(dest_dir, name) + if os.path.exists(dst): + if _same_file(dst, size, sha256): + return dst + raise RuntimeError(f"{dst} already exists with different content; not overwritten") + if move: + try: + os.rename(src, dst) + _chown_like(dst, dest_dir) + return dst + except OSError as ex: + if ex.errno != 18: # EXDEV: other filesystem, copy instead + raise + fd, tmp = tempfile.mkstemp(dir=dest_dir, prefix=".tmp-") + os.close(fd) + try: + shutil.copyfile(src, tmp) + if not _same_file(tmp, size, sha256): + raise RuntimeError(f"copy of {name} to {dest_dir} failed verification") + os.chmod(tmp, 0o644) + _chown_like(tmp, dest_dir) + os.replace(tmp, dst) + finally: + if os.path.exists(tmp): + os.remove(tmp) + return dst + + +def _write_like(path, data, ts, tag="pre-webupdate"): + """Back up path to path.bak--, then atomically replace it with + data (bytes), keeping owner and mode. Returns the backup path.""" + real = os.path.realpath(path) + backup = f"{path}.bak-{ts}-{tag}" + n = 1 + while os.path.lexists(backup): # never overwrite an earlier backup + n += 1 + backup = f"{path}.bak-{ts}-{tag}-{n}" + shutil.copy2(real, backup) + st = os.stat(real) + folder = os.path.dirname(real) + fd, tmp = tempfile.mkstemp(dir=folder, prefix=".tmp-") + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp, st.st_mode & 0o7777) + try: + os.chown(tmp, st.st_uid, st.st_gid) + os.chown(backup, st.st_uid, st.st_gid) + except OSError: + pass + os.replace(tmp, real) + finally: + if os.path.exists(tmp): + os.remove(tmp) + return backup + + +def dump_map(entries, raw): + """Serialise a map the way the originals are written: indent 4, UTF-8, + a trailing newline only if the original had one, BOM only if it had one.""" + text = json.dumps(entries, indent=4, ensure_ascii=False) + if raw.endswith(b"\n"): + text += "\n" + data = text.encode("utf-8") + return (b"\xef\xbb\xbf" + data) if raw.startswith(b"\xef\xbb\xbf") else data + + +def history(): + return uc.read_json(_path("applied.json"), []) or [] + + +class _Busy: + """Non-blocking exclusive lock so two applies/rollbacks (a double click, + two tabs) never interleave their reads and rewrites of the same maps.""" + + def __enter__(self): + self.fd = os.open(_path(".apply.lock"), os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(self.fd) + raise RuntimeError("another apply or rollback is already running; try again " + "when it has finished") from None + return self + + def __exit__(self, *exc): + fcntl.flock(self.fd, fcntl.LOCK_UN) + os.close(self.fd) + return False + + +def apply(keys): + """Install the staged items named by keys. Returns a report dict.""" + with _Busy(): + return _apply(keys) + + +def _apply(keys): + ts = _stamp() + report = {"id": ts, "at": uc.now(), "drivers": [], "bios": [], "skipped": [], + "errors": [], "backups": {}, "after": {}} + maps = read_maps(report["errors"]) + changed_maps = set() + items = _ready(list(dict.fromkeys(keys))) + staged_items = {} # only the items this apply changed are written back + + # Driver zips first: a map only changes once its zip is in place. + for it in [i for i in items if i["kind"] == "driver"]: + todo = [] + for cur in it.get("replaces", []): + if "nvidia" in cur.lower() and not it.get("nvidia_infs"): + report["skipped"].append(f"{cur}: has NVIDIA drivers, Dell pack {it['zip']} has none") + continue + hits = [(img, e) for img, m in maps.items() for e in m["entries"] + if isinstance(e, dict) and entry_file(e) == cur] + if not hits: + report["skipped"].append(f"{cur}: no map entry uses it any more") + continue + todo.append((cur, hits)) + if not todo: + continue + placed = [] + try: + if not _CMD_SAFE.fullmatch(it["zip"]): + raise RuntimeError("unsafe zip name") + dirs = [] + for _, hits in todo: + for _, e in hits: + rel = entry_dir(e) + if ".." in PurePosixPath(rel).parts: + raise RuntimeError(f"destinationDir {rel} leaves the share") + d = resolve_dir_ci(config.SHARED_DIR, rel, create=True) + if d is None: + raise RuntimeError(f"cannot resolve {rel}") + if os.path.realpath(d) not in {os.path.realpath(x) for x in dirs}: + dirs.append(d) + src, move = it["file"], True + for d in dirs: + p = _place(src, d, it["zip"], it["size_file"], it["sha256_file"], move) + placed.append(p) + src, move = p, False # later directories get verified copies + except Exception as ex: + report["errors"].append(f"{it['zip']}: {ex}") + if placed and not os.path.exists(it["file"]): + # The staged zip was renamed onto the share before a later + # folder failed: point the item there so a retry can finish. + staged_items.setdefault(it["key"], {})["file"] = placed[0] + continue + for cur, hits in todo: + for img, e in hits: + for k in ("fileName", "FileName"): + if k in e: + e[k] = it["zip"] + e["url"] = it["url"] + e["hash"] = it["sha256_file"] + e["size"] = it["size_file"] + changed_maps.add(img) + report["drivers"].append({"old": cur, "new": it["zip"], "placed": placed, + "images": sorted({img for img, _ in hits})}) + staged_items.setdefault(it["key"], {}).update(status="applied", applied=ts, placed=placed) + + # One write per real file: if an image's map is a symlink to another's, + # writing it twice would back up the already-updated copy the second time. + written = set() + for img in sorted(changed_maps, key=lambda i: (os.path.islink(maps[i]["path"]), i)): + m = maps[img] + real = os.path.realpath(m["path"]) + if real in written: + continue + written.add(real) + data = dump_map(m["entries"], m["raw"]) + try: + report["backups"][m["path"]] = _write_like(m["path"], data, ts) + report["after"][m["path"]] = hashlib.sha256(data).hexdigest() + except OSError as ex: + report["errors"].append(f"{img} HardwareDriver.json not written ({ex}); its zip is on " + "the share but this image still uses the old file") + + # BIOS: only the file|version fields of its lines; the exe goes into + # BIOS_DIR only when at least one line will point at it. + bios_items = [i for i in items if i["kind"] == "bios"] + if bios_items: + mpath = models_txt_path() + try: + with open(mpath, "rb") as f: + raw = f.read() + text = raw.decode("ascii") + except (OSError, UnicodeDecodeError) as ex: + report["errors"].append(f"models.txt: {ex}") + text = None + if text is not None: + split = text.split("\n") + lines_by_no = {n: t.rstrip("\r") for n, t in enumerate(split)} + edits, done = {}, [] + for it in bios_items: + if not bios_name_ok(it["name"], it["ver"]): + report["errors"].append(f"{it['name']}: file name or version not safe for " + "check-bios.cmd; not applied") + continue + mine = {} + for ln in it.get("lines", []): + n = _find_line(lines_by_no, ln) + if n is None: + report["skipped"].append(f"models.txt line '{ln['line']}' changed since the check") + continue + mine[n] = _new_line(lines_by_no[n], it["name"], it["ver"]) + if not mine: + continue + try: + placed = _place(it["file"], config.BIOS_DIR, it["name"], it["size_file"], + it["sha256_file"], True) + except Exception as ex: + report["errors"].append(f"{it['name']}: {ex}") + continue + edits.update(mine) + done.append((it, placed, mine)) + if edits: + for n, new in edits.items(): + cr = "\r" if split[n].endswith("\r") else "" + split[n] = new + cr + data = "\n".join(split).encode("ascii") + try: + report["backups"][mpath] = _write_like(mpath, data, ts) + report["after"][mpath] = hashlib.sha256(data).hexdigest() + except OSError as ex: + report["errors"].append(f"models.txt not written ({ex}); the BIOS files are in " + "place but no line points at them yet") + for it, placed, _ in done: # keep them applicable for a retry + staged_items.setdefault(it["key"], {})["file"] = placed + done = [] + for it, placed, mine in done: + for n, new in mine.items(): + report["bios"].append({"old": lines_by_no[n], "new": new}) + staged_items.setdefault(it["key"], {}).update(status="applied", applied=ts, + placed=[placed]) + + # Merge field by field into a fresh read: a download job may have + # written other items while this ran. + state = staged_state() + for k, v in staged_items.items(): + if k in state.get("items", {}): + state["items"][k].update(v) + _save_staged(state) + if report["drivers"] or report["bios"] or report["errors"]: + hist = history() + hist.insert(0, report) + uc.write_json(_path("applied.json"), hist[:50]) + _audit("DELL_APPLY", f"{len(report['drivers'])} driver replacement(s), " + f"{len(report['bios'])} BIOS line(s), {len(report['skipped'])} skipped, " + f"{len(report['errors'])} error(s)") + _invalidate() + return report + + +def rollback(apply_id): + """Restore the maps and models.txt saved before apply apply_id. A file + changed again since is left alone and reported. Placed zips/exes stay.""" + with _Busy(): + return _rollback(apply_id) + + +def _rollback(apply_id): + rec = next((h for h in history() if h.get("id") == apply_id), None) + if rec is None: + raise ValueError("unknown apply") + ts = _stamp() + restored, refused = [], [] + for path, backup in rec.get("backups", {}).items(): + try: + with open(path, "rb") as f: + now_hash = hashlib.sha256(f.read()).hexdigest() + if now_hash != rec.get("after", {}).get(path): + refused.append(f"{path} changed since that apply") + continue + with open(backup, "rb") as f: + data = f.read() + _write_like(path, data, ts, tag="pre-rollback") + restored.append(path) + except OSError as ex: + refused.append(f"{path}: {ex}") + hist = history() + for h in hist: + if h.get("id") == apply_id: + h["rolled_back"] = uc.now() if restored else h.get("rolled_back") + uc.write_json(_path("applied.json"), hist) + _audit("DELL_ROLLBACK", f"{apply_id}: restored {len(restored)}, refused {len(refused)}") + _invalidate() + return restored, refused + + +def _audit(action, detail): + try: + from services.audit import audit + audit(action, detail) + except Exception: + pass # audit log not writable (local tests): the history file still has it + + +def _invalidate(): + try: + from services import hardware_coverage + hardware_coverage.invalidate() + except Exception: + pass + + +# --- hints for /hardware ---------------------------------------------------------- + +def update_hints(): + """{"drivers": {current zip (lower): Dell version}, "bios": {current BIOS + file (lower): Dell version}} for rows of the last scan with an update. + Keyed by file so /hardware can match what each model actually resolves + to (first matching map entry / models.txt line) without re-matching.""" + scan = last_scan() or {} + out = {"drivers": {}, "bios": {}} + for r in scan.get("drivers", []): + if r.get("status") == "update" and r.get("current"): + out["drivers"][r["current"].lower()] = (r.get("pack") or {}).get("ver", "") + for r in scan.get("bios", []): + if r.get("status") == "update" and r.get("current"): + cands = r.get("candidates") or [] + pick = r.get("pick") or 0 + ver = cands[pick].get("version", "") if 0 <= pick < len(cands) else "" + out["bios"].setdefault(r["current"].lower(), ver) + return out + + +# --- job entry point -------------------------------------------------------------- + +def start(kind): + """Start the check or download job (systemd-run). The two never overlap.""" + other = "download" if kind == "check" else "check" + if uc.job_running(UNITS[other]): + raise RuntimeError(f"the {other} job is still running") + if kind == "check": + if uc.job_running(UNITS["check"]): + raise RuntimeError("a check is already running") + uc.write_json(_path("check.json"), {"state": "running", "started": uc.now(), + "step": "Starting"}) + try: + uc.start_job(UNITS[kind], ["services.dell_updates", kind]) + except Exception as ex: + if kind == "check": # else the page would show a check that never ran + _check_progress(state="error", finished=uc.now(), error=str(ex)[:400]) + raise + + +def main(argv): + if not argv or argv[0] not in ("check", "download", "apply"): + print("usage: python3 -m services.dell_updates check|download|apply KEY...", file=sys.stderr) + return 2 + if argv[0] == "check": + check(fetch="--offline" not in argv) + elif argv[0] == "download": + download() + else: + print(json.dumps(apply(argv[1:]), indent=1)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/webapp/services/driver_lint.py b/webapp/services/driver_lint.py new file mode 100644 index 0000000..747dc92 --- /dev/null +++ b/webapp/services/driver_lint.py @@ -0,0 +1,321 @@ +"""Lint an image's HardwareDriver.json against PESetup's real driver matcher. + +A port of scripts/lint-driver-catalogue.py for the webapp (the script is +left as it is: it still gates deploys from a workstation). Same matcher, +same checks, with three differences that come from running on the server: + + * zips are looked up on the share directly, and a zip found only with + different letter case is a WARN, not an ERROR, when WinPE still finds + it: WinPE reads the share over SMB and Samba matches each folder name + case-insensitively for Windows clients (default "case sensitive = + auto"). It stays an ERROR when a folder with the exact spelling exists + and the zip sits in its other-case twin; + * a missing (null) modelswminame or family is CRITICAL, not just "no + tokens": the C# calls .Split / .ToLower on them, which throws for every + model the family filter lets through; + * "win10-first": a Win11 entry's model is answered by a Win10 entry + earlier in the list. Every GE image deploys Windows 11 and PESetup + has no OS filter, so those PCs get Windows 10 drivers. + +Positions in messages are 1-based (#5 = fifth entry), as on the Drivers tab. + +lint_entries(entries, image_type) -> findings, each a dict: + {"level", "check", "message", "index" (0-based or None), + "indexes" (every 0-based entry the finding is about)} +lint_image(image_type) reads the map from disk (used by the Overview tab). +""" + +import json +import os +from pathlib import PurePosixPath + +from services import fs, image_pages + +CRITICAL, ERROR, WARN, INFO = "CRITICAL", "ERROR", "WARN", "INFO" +LEVELS = (CRITICAL, ERROR, WARN, INFO) +LEVEL_ORDER = {lvl: i for i, lvl in enumerate(LEVELS)} + +# Short names for the checks, shown as badges on the Drivers tab. +CHECK_LABELS = { + "virtual-platform": "virtual platform", + "null-tokens": "no modelswminame", + "null-family": "no family", + "empty-token": "empty token", + "token-whitespace": "token whitespace", + "no-tokens": "no tokens", + "family-mismatch": "family mismatch", + "no-filename": "no file name", + "missing-zip": "zip missing", + "case-mismatch": "zip case differs", + "win10-first": "Win10 entry wins", + "shadowed": "shadowed", + "duplicate-token": "duplicate token", + "bad-entry": "not an object", +} + + +# --- PESetup's matcher (docs/PESETUP-INTERNALS.md, GetDriverByModel) --------- + +def model_family(model): + """Three Contains() tests, the last hit wins, "" when none hit.""" + upper = model.upper() + family = "" + if "LATITUDE" in upper: + family = "Latitude" + if "OPTIPLEX" in upper: + family = "Optiplex" + if "PRECISION" in upper: + family = "Precision" + return family + + +def entry_tokens(entry): + """modelswminame split on comma. NOT trimmed: the C# does not trim. + A missing (null) modelswminame gives no tokens here; PESetup throws on + it instead, which the null-tokens check reports.""" + if entry.get("modelswminame") is None: + return [] + return str(entry.get("modelswminame")).split(",") + + +def entry_matches(entry, model): + family = model_family(model) + if family and family.lower() not in str(entry.get("family") or "").lower(): + return False + # model.ToLower().Contains(token.ToLower()); "" is in every string. + return any(t.lower() in model.lower() for t in entry_tokens(entry)) + + +def select_driver(entries, model): + """First match wins, like GetDriverByModel: (index, entry) or (None, None).""" + for i, entry in enumerate(entries): + if entry_matches(entry, model): + return i, entry + return None, None + + +# --- helpers ------------------------------------------------------------------- + +def label(entry, index): + name = entry.get("modelsfriendlyname") or entry.get("modelswminame") or "?" + return "#%d %s" % (index + 1, name) + + +def _smb_lookup(root, rel): + """Resolve rel the way Samba does for a Windows client: per folder, the + exact name when it exists, else the one name that differs only in case. + No backtracking: once a folder is picked, a file under its other-case + twin is not found (Dell_11 has held both Optiplex and OptiPlex).""" + cur = root + for part in PurePosixPath(rel).parts: + nxt = os.path.join(cur, part) + if not os.path.lexists(nxt): + try: + names = [n for n in os.listdir(cur) if n.lower() == part.lower()] + except OSError: + return None + if len(names) != 1: + return None + nxt = os.path.join(cur, names[0]) + cur = nxt + return cur if os.path.isfile(cur) else None + + +def zip_state(image_type, entry): + """Where an entry's zip is: {"rel", "status", "path"}. rel is relative + to the image root (Deploy/Out-of-box Drivers/...). status: + ok the exact path exists + case only with other letter case, and WinPE (SMB) still finds it + trap on disk under another-case folder that SMB does not pick + missing nowhere + none the entry names no file""" + rel = image_pages.hardware_coverage._zip_relpath(entry) + if not rel: + return {"rel": "", "status": "none", "path": None} + if ".." in PurePosixPath(rel).parts: + # Never look outside the image tree: /drivers/check takes any + # posted entry, and no real entry climbs out with "..". + return {"rel": rel, "status": "missing", "path": None} + root = fs.image_root(image_type) + exact = os.path.join(root, rel) + if os.path.isfile(exact): + return {"rel": rel, "status": "ok", "path": exact} + smb = _smb_lookup(root, rel) + if smb: + return {"rel": rel, "status": "case", "path": smb, "actual": os.path.relpath(smb, root)} + found = image_pages.zip_path(image_type, entry) + if found: + return {"rel": rel, "status": "trap", "path": found, "actual": os.path.relpath(found, root)} + return {"rel": rel, "status": "missing", "path": None} + + +def _add(out, level, check, message, index=None, indexes=None): + idx = list(indexes) if indexes is not None else ([] if index is None else [index]) + out.append({"level": level, "check": check, "message": message, + "index": index, "indexes": idx}) + + +# --- checks -------------------------------------------------------------------- + +def lint_entries(entries, image_type=None, zips=None): + """Every finding for a list of entries. image_type enables the zip + checks; zips (a list of zip_state results, one per entry) saves + looking them up again.""" + out = [] + rows = [e if isinstance(e, dict) else {} for e in entries] + for i, e in enumerate(entries): + if not isinstance(e, dict): + _add(out, CRITICAL, "bad-entry", + "#%d is not a JSON object; PESetup cannot read the map." % (i + 1), i) + + for i, entry in enumerate(rows): + tag = label(entry, i) + + # Virtual platform hijack: one such entry anywhere flips PESetup into + # virtual-only mode and every physical machine fails. + manufacturer = str(entry.get("manufacturer") or "") + if "virtual platform" in manufacturer.lower(): + _add(out, CRITICAL, "virtual-platform", + "%s manufacturer is %r. PESetup then takes the first entry as the driver and " + "REQUIRES the machine to look virtual, so every physical PC fails." % (tag, manufacturer), i) + + # Null fields the C# dereferences without a check. + if entry.get("modelswminame") is None: + _add(out, CRITICAL, "null-tokens", + "%s has no modelswminame. PESetup calls Split() on it and throws for every " + "model the family filter lets through, so driver selection fails." % tag, i) + if entry.get("family") is None: + _add(out, CRITICAL, "null-family", + "%s has no family. PESetup calls ToLower() on it for every Latitude, OptiPlex " + "and Precision model and throws." % tag, i) + + # Empty token: Contains("") is true for every model. An empty + # modelswminame is one empty token, so it matches everything too. + tokens = entry_tokens(entry) + if tokens == [""]: + _add(out, CRITICAL, "empty-token", + "%s modelswminame is empty. Split gives one empty token, which matches every " + "model, so this entry takes over the map." % tag, i) + elif any(t == "" for t in tokens): + _add(out, CRITICAL, "empty-token", + "%s modelswminame %r has an EMPTY token (trailing or doubled comma). An empty " + "token matches every model, so this entry takes over the map." + % (tag, entry.get("modelswminame")), i) + + # Whitespace around a token is kept by Split(','), so it must be in the model. + for token in tokens: + if token and token != token.strip(): + _add(out, ERROR, "token-whitespace", + "%s token %r carries whitespace. PESetup does not trim, so the model " + "string must contain the space too; this token will usually miss." % (tag, token), i) + + if tokens and tokens != [""] and not str(entry.get("modelswminame")).strip(): + _add(out, ERROR, "no-tokens", + "%s has an empty modelswminame, so no model can ever select it." % tag, i) + + # The family filter runs on the MODEL first, then wants entry.family + # to contain it: a token implying another family can never match. + family_field = str(entry.get("family") or "") + for token in tokens: + token = token.strip() + implied = model_family(token) if token else "" + if implied and implied.lower() not in family_field.lower(): + _add(out, ERROR, "family-mismatch", + "%s token %r implies family %s, but family is %r. The filter runs before " + "the token test, so this token can never match." % (tag, token, implied, family_field), i) + + # The referenced zip. + if not str(entry.get("fileName") or entry.get("FileName") or ""): + _add(out, ERROR, "no-filename", + "%s has no fileName, so PESetup has nothing to unzip." % tag, i) + elif image_type: + z = zips[i] if zips is not None else zip_state(image_type, entry) + if z["status"] == "missing": + _add(out, ERROR, "missing-zip", + "%s references %s, which is not on the share. A driver miss is only a " + "warning in PESetup, so imaging finishes with no drivers." % (tag, z["rel"]), i) + elif z["status"] == "case": + _add(out, WARN, "case-mismatch", + "%s references %s but the share holds %s. WinPE finds it (Samba matches " + "case-insensitively), Linux tools and the lint script do not." + % (tag, z["rel"], z.get("actual", "")), i) + elif z["status"] == "trap": + _add(out, ERROR, "case-mismatch", + "%s references %s but the zip is at %s. A folder with the exact spelling " + "exists and does not hold it, so WinPE will not find it." + % (tag, z["rel"], z.get("actual", "")), i) + + # Shadowing: replay each entry's own tokens through the matcher. When a + # Win10 entry answers a Win11 entry's model, that is worse than dead + # weight: Win11 PCs get Windows 10 drivers. + os_of = [image_pages.driver_os(e) for e in rows] + for i, entry in enumerate(rows): + for token in entry_tokens(entry): + token = token.strip() + if not token: + continue + win, _ = select_driver(rows, token) + if win is None or win == i: + continue + if os_of[i] == "Win11" and os_of[win] == "Win10": + _add(out, ERROR, "win10-first", + "%s token %r is answered by %s, a Windows 10 pack earlier in the list. " + "PESetup has no OS filter, so Windows 11 PCs get Windows 10 drivers. " + "Move the Win11 entry above it." % (label(entry, i), token, label(rows[win], win)), + i, indexes=[i, win]) + else: + _add(out, WARN, "shadowed", + "%s token %r is answered by %s instead (first match wins)." + % (label(entry, i), token, label(rows[win], win)), i) + + # One token claimed by several entries: the first one wins whatever the OS. + by_token = {} + for i, entry in enumerate(rows): + for token in entry_tokens(entry): + token = token.strip().lower() + if token: + by_token.setdefault(token, []) + if i not in by_token[token]: + by_token[token].append(i) + for token, idxs in sorted(by_token.items()): + if len(idxs) > 1: + detail = ", ".join("%s (%s, osId %s)" % (label(rows[i], i), os_of[i], rows[i].get("osId")) + for i in idxs) + _add(out, WARN, "duplicate-token", + "token %r is claimed by %d entries: %s. osId is not part of the match, so %s " + "wins whatever OS is applied." % (token, len(idxs), detail, label(rows[idxs[0]], idxs[0])), + None, indexes=idxs) + + out.sort(key=lambda f: (LEVEL_ORDER[f["level"]], f["indexes"][0] if f["indexes"] else -1)) + return out + + +def counts(findings): + """{"total", "CRITICAL": n, "ERROR": n, "WARN": n, "INFO": n}.""" + c = {lvl: 0 for lvl in LEVELS} + for f in findings: + c[f["level"]] += 1 + c["total"] = sum(c[lvl] for lvl in LEVELS) + return c + + +def map_path(image_type): + return image_pages.control_file(image_type, "HardwareDriver.json") + + +def lint_image(image_type): + """Findings for the map on disk. [] when the image has no map; one + CRITICAL finding when it does not parse or is not a list.""" + path = map_path(image_type) + if not os.path.isfile(path): + return [] + try: + with open(path, encoding="utf-8-sig") as fh: + entries = json.load(fh) + except (OSError, ValueError) as exc: + return [{"level": CRITICAL, "check": "parse", "index": None, "indexes": [], + "message": "HardwareDriver.json does not parse: %s" % exc}] + if not isinstance(entries, list): + return [{"level": CRITICAL, "check": "parse", "index": None, "indexes": [], + "message": "HardwareDriver.json is a %s, expected a list" % type(entries).__name__}] + return lint_entries(entries, image_type) diff --git a/webapp/services/driver_map.py b/webapp/services/driver_map.py new file mode 100644 index 0000000..a8bcff7 --- /dev/null +++ b/webapp/services/driver_map.py @@ -0,0 +1,416 @@ +"""The Drivers tab's view of an image's HardwareDriver.json. + +PESetup reads Deploy/Control/HardwareDriver.json and takes the FIRST entry +whose modelswminame token is in the PC's WMI model (no OS filter), so the +list order matters and is kept exactly as the user arranges it. + +The page edits the whole map in the browser and posts it back as a list of +entries; every entry that came from the file carries "_orig" (its 0-based +position when the page was loaded), so the server can tell an edit from a +move, an addition or a removal. Keys starting with "_" are never written. + + rows(image_type, entries) per-entry facts for the table (zip on disk, + OS, lint findings, Dell update hint) + orphans(image_type, entries) zips under the image's driver folder that + this map does not reference + staged(body) validate a posted map: (entries, origs) + changes(old, new, origs) per-entry summary + JSON diff for the review + delete_orphans(image_type, rels) zips no map, undo or rollback uses + save_upload(image_type, file) an uploaded zip, temp file + rename + +Writes go through services/control_files (format kept, backup, undo). +""" + +import difflib +import json +import os +import tempfile + +import config +from services import control_files, driver_lint, fs, image_pages + +FILE_NAME = "HardwareDriver.json" +DRIVERS_SUBDIR = "Out-of-box Drivers" +DEST_PREFIX = "*destinationdir*\\Deploy\\Out-of-box Drivers\\" + +# Keys the C# reads as strings; a posted entry must not turn them into +# numbers or objects. +STRING_KEYS = ("manufacturer", "product", "family", "modelswminame", "modelsfriendlyname", + "fileName", "destinationDir", "FileName", "DestinationDir", "osId", "models") + + +def map_path(image_type): + return image_pages.control_file(image_type, FILE_NAME) + + +def load(image_type): + """(entries, error). entries is [] and error a message when the file is + missing, does not parse or is not a list.""" + path = map_path(image_type) + if not os.path.isfile(path): + return [], None + try: + with open(path, encoding="utf-8-sig") as fh: + data = json.load(fh) + except (OSError, ValueError) as exc: + return [], f"{FILE_NAME} does not parse: {exc}" + if not isinstance(data, list): + return [], f"{FILE_NAME} is a {type(data).__name__}, expected a list" + # The page edits entries as objects; a string or null entry would be + # turned into an object (or crash the page), so such a map is read-only. + bad = [i + 1 for i, e in enumerate(data) if not isinstance(e, dict)] + if bad: + return [], (f"{FILE_NAME} entry #{bad[0]} is not a JSON object " + f"({len(bad)} such entr{'y' if len(bad) == 1 else 'ies'})") + return data, None + + +def file_sha(image_type): + return control_files.file_sha256(map_path(image_type)) + + +def _stat(path): + try: + st = os.stat(path) + return st.st_size, st.st_mtime + except (OSError, TypeError): + return None, None + + +def _dell_hints(): + try: + from services import dell_updates + return dell_updates.update_hints()["drivers"] + except Exception: # no scan yet or the module failed: no hints + return {} + + +def rows(image_type, entries): + """(rows, findings, counts) for a list of entries (from the file or + staged in the page). One row per entry, in order: + {"os", "zip": {"status", "rel", "actual", "size", "mtime"}, + "findings": [finding, ...], "dell": newer Dell version or ""}""" + dicts = [e if isinstance(e, dict) else {} for e in entries] + zips = [driver_lint.zip_state(image_type, e) for e in dicts] + findings = driver_lint.lint_entries(entries, image_type, zips=zips) + per = [[] for _ in entries] + for f in findings: + for i in f["indexes"]: + if 0 <= i < len(per): + per[i].append({"level": f["level"], "check": f["check"], "message": f["message"], + "label": driver_lint.CHECK_LABELS.get(f["check"], f["check"])}) + hints = _dell_hints() + out = [] + for i, e in enumerate(dicts): + z = zips[i] + size, mtime = _stat(z["path"]) + name = str(e.get("fileName") or e.get("FileName") or "") + out.append({ + "os": image_pages.driver_os(e), + "zip": {"status": z["status"], "rel": z["rel"], "actual": z.get("actual", ""), + "size": size, "mtime": mtime}, + "findings": per[i], + "dell": hints.get(name.lower(), ""), + }) + return out, findings, driver_lint.counts(findings) + + +# --- orphans ------------------------------------------------------------------------ + +def _drivers_root(image_type): + """Real path of the image's Out-of-box Drivers folder (a symlink into + _shared on the server), or None.""" + path = os.path.join(fs.deploy_path(image_type), DRIVERS_SUBDIR) + real = os.path.realpath(path) + return real if os.path.isdir(real) else None + + +def _share_images(): + """Every image whose map can point into the shared driver tree: the + registered ones plus any other folder on the share with a map (an image + dropped from the registry can still be booted).""" + images = list(config.IMAGE_TYPES) + try: + names = sorted(os.listdir(config.SAMBA_SHARE)) + except OSError: + names = [] + for name in names: + if (not name.startswith("_") and name not in images + and os.path.isfile(map_path(name))): + images.append(name) + return images + + +def _restore_points(): + """[(image, label, map file)] for maps the app can still put back: the + backup behind each image's Undo last save, and the map backups of every + Dell update that can be rolled back. A zip those maps use must not be + deleted, or the undo / rollback restores an entry with no zip.""" + out = [] + for image in _share_images(): + rec = control_files.last_save(image, "drivers") or {} + for f in rec.get("files", []): + if f.get("backup") and os.path.basename(f.get("path", "")) == FILE_NAME: + out.append((image, f"{image} (undo of last save)", f["backup"])) + try: + from services import dell_updates + history = dell_updates.history() + except Exception: # no Dell history yet or the module failed + history = [] + for rec in history: + if rec.get("rolled_back"): + continue + for path, bak in (rec.get("backups") or {}).items(): + if os.path.basename(path) != FILE_NAME or not bak: + continue + image = os.path.relpath(path, config.SAMBA_SHARE).split(os.sep)[0] + out.append((image, f"{image} (Dell update rollback)", bak)) + return out + + +def _entries_of(path): + try: + with open(path, encoding="utf-8-sig") as fh: + data = json.load(fh) + except (OSError, ValueError): + return [] + return [e for e in data if isinstance(e, dict)] if isinstance(data, list) else [] + + +def _referenced(): + """{real zip path: [who uses it]}: every image's map, plus the maps an + undo or a Dell rollback would restore (see _restore_points).""" + refs = {} + sources = [(image, image, map_path(image)) for image in _share_images()] + for image, label, path in sources + _restore_points(): + for e in _entries_of(path): + p = image_pages.zip_path(image, e) + if p: + refs.setdefault(os.path.realpath(p), []).append(label) + return refs + + +def orphans(image_type, refs=None): + """Zips under this image's driver folder that its map does not use. + used_by lists the other images that do use it (deleting those would + break them, so the page only allows Add to map).""" + root = _drivers_root(image_type) + if not root: + return [] + refs = _referenced() if refs is None else refs + out = [] + # followlinks=False: Dell_11/OptiPlex can be a symlink to Optiplex, and + # following it would list every zip there twice. + for dirpath, dirnames, filenames in os.walk(root): + dirnames.sort() + for fn in sorted(filenames): + if not fn.lower().endswith(".zip"): + continue + full = os.path.join(dirpath, fn) + users = sorted(set(refs.get(os.path.realpath(full), []))) + if image_type in users: + continue + rel = os.path.relpath(full, root).replace(os.sep, "/") + folder = os.path.dirname(rel) + size, mtime = _stat(full) + out.append({ + "name": fn, + "rel": rel, + "folder": folder, + "dest": DEST_PREFIX.rstrip("\\") + ("\\" + folder.replace("/", "\\") if folder else ""), + "size": size, + "mtime": mtime, + "used_by": users, + }) + return out + + +def delete_orphans(image_type, rels): + """Delete zips (paths relative to the driver folder) that NO image's + map references. Returns {"removed": [...], "refused": [(rel, why)]}.""" + root = _drivers_root(image_type) + removed, refused = [], [] + if not root: + return {"removed": removed, "refused": [(r, "no driver folder") for r in rels]} + refs = _referenced() + for rel in rels: + rel = str(rel or "") + full = os.path.realpath(os.path.join(root, rel)) + if not full.startswith(root + os.sep) or not full.lower().endswith(".zip"): + refused.append((rel, "not a zip in the driver folder")) + elif os.path.islink(os.path.join(root, rel)): + # Unlinking the resolved path would delete the zip it points at. + refused.append((rel, "a symlink; remove it by hand")) + elif not os.path.isfile(full): + refused.append((rel, "not found")) + elif refs.get(full): + refused.append((rel, "used by " + ", ".join(sorted(set(refs[full]))))) + else: + try: + os.unlink(full) + removed.append(rel) + except OSError as exc: + refused.append((rel, str(exc))) + return {"removed": removed, "refused": refused} + + +def save_upload(image_type, upload, overwrite=False): + """Store an uploaded driver .zip at the top of the image's driver folder. + Written to a temp file next to it and renamed, so a PC copying the old + file never reads a half-written one; owner and group follow the folder + (the app runs as root, the share is pxe:pxe). Overwriting a zip any map, + undo or rollback uses is refused: its entry's hash and size would no + longer match. Nothing is added to the map (Add to map does that, through + the review step). Returns {"filename", "path", "replaced"}.""" + from services import images # the existing upload name sanitizer + name = images._safe_filename(getattr(upload, "filename", "")) + if not name.lower().endswith(".zip"): + raise ValueError("Driver upload must be a .zip file") + root = _drivers_root(image_type) + if not root: + root = os.path.join(fs.deploy_path(image_type), DRIVERS_SUBDIR) + os.makedirs(root, exist_ok=True) + dst = os.path.join(root, name) + replaced = os.path.lexists(dst) + if replaced: + if not overwrite: + raise FileExistsError(f"{name} already exists in {DRIVERS_SUBDIR}/") + if os.path.islink(dst) or not os.path.isfile(dst): + raise ValueError(f"{name} in {DRIVERS_SUBDIR}/ is not a plain file; replace it by hand") + users = sorted(set(_referenced().get(os.path.realpath(dst), []))) + if users: + raise ValueError(f"{name} is in use ({', '.join(users)}); upload it under a new name " + "and point the entry at it instead") + fd, tmp = tempfile.mkstemp(dir=root, prefix=".webui-upload-", suffix=".part") + try: + with os.fdopen(fd, "wb") as fh: + upload.save(fh) + fh.flush() + os.fsync(fh.fileno()) + os.chmod(tmp, 0o664) + try: + st = os.stat(root) + os.chown(tmp, st.st_uid, st.st_gid) + except OSError: + pass # not root (local tests) + os.replace(tmp, dst) + except BaseException: + if os.path.exists(tmp): + os.remove(tmp) + raise + return {"filename": name, "path": dst, "replaced": replaced} + + +# --- staged map ----------------------------------------------------------------------- + +def staged(body, old_len): + """Validate a posted {"entries": [...]}. Returns (entries, origs, moved): + entries cleaned of "_" keys, origs[i] = position in the file the entry + came from (or None for a new one), moved[i] = the user moved it with + the arrows ("_moved"; only used to word the review). Raises ValueError.""" + raw = body.get("entries") if isinstance(body, dict) else None + if not isinstance(raw, list) or not all(isinstance(e, dict) for e in raw): + raise ValueError("expected entries: a list of objects") + entries, origs, moved, seen = [], [], [], set() + for n, e in enumerate(raw, 1): + o = e.get("_orig") + if isinstance(o, bool) or not isinstance(o, int) or not 0 <= o < old_len or o in seen: + o = None + else: + seen.add(o) + clean = {k: v for k, v in e.items() if not str(k).startswith("_")} + for k in STRING_KEYS: + if k in clean and clean[k] is not None and not isinstance(clean[k], str): + raise ValueError(f"entry #{n}: {k} must be text") + if "aOsIds" in clean and not (isinstance(clean["aOsIds"], list) + and all(isinstance(x, str) for x in clean["aOsIds"])): + raise ValueError(f"entry #{n}: aOsIds must be a list of text") + entries.append(clean) + origs.append(o) + moved.append(bool(e.get("_moved"))) + return entries, origs, moved + + +def _dump(entry): + return json.dumps(entry, indent=4, ensure_ascii=False) + + +def _kept_in_order(seq, weights): + """Indexes (into seq) of the heaviest increasing run of values: the + entries that kept their relative order; the rest were moved. When the + user swaps two neighbours either one could be "the moved one", so the + entries the user did not touch weigh more and stay in the run.""" + n = len(seq) + if not n: + return set() + best = list(weights) + prev = [-1] * n + for i in range(n): + for j in range(i): + if seq[j] < seq[i] and best[j] + weights[i] > best[i]: + best[i], prev[i] = best[j] + weights[i], j + i = max(range(n), key=lambda k: best[k]) + keep = set() + while i != -1: + keep.add(i) + i = prev[i] + return keep + + +def changes(old, new, origs, user_moved=None): + """What a save would do, entry by entry, for the review step: + [{"kind": "added"|"removed"|"edited"|"moved"|"edited+moved", + "label", "old_pos", "new_pos" (1-based or None), "file" (moves only), + "fields", "diff"}] + and a summary {"added", "removed", "edited", "moved"}.""" + out = [] + kept = [(i, o) for i, o in enumerate(origs) if o is not None] + user_moved = user_moved or [False] * len(new) + in_order = _kept_in_order([o for _, o in kept], + [1 if user_moved[i] else 1000 for i, _ in kept]) + moved = {kept[k][0] for k in range(len(kept)) if k not in in_order} + summary = {"added": 0, "removed": 0, "edited": 0, "moved": 0} + + for i, (entry, o) in enumerate(zip(new, origs)): + lbl = driver_lint.label(entry, i) + if o is None: + summary["added"] += 1 + out.append({"kind": "added", "label": lbl, "old_pos": None, "new_pos": i + 1, + "fields": list(entry.keys()), + "diff": _udiff("", _dump(entry) + "\n", "(new)", f"#{i + 1}")}) + continue + before = old[o] if isinstance(old[o], dict) else {} + edited = _dump(before) != _dump(entry) + was_moved = i in moved + if not edited and not was_moved: + continue + kind = "edited+moved" if edited and was_moved else ("edited" if edited else "moved") + if edited: + summary["edited"] += 1 + if was_moved: + summary["moved"] += 1 + fields = sorted({k for k in set(before) | set(entry) if before.get(k) != entry.get(k)} + | ({"(key order)"} if edited and list(before) != list(entry) + and set(before) == set(entry) else set())) + out.append({"kind": kind, "label": lbl, "old_pos": o + 1, "new_pos": i + 1, + "file": str(entry.get("fileName") or entry.get("FileName") or ""), + "fields": fields, + "diff": _udiff(_dump(before) + "\n", _dump(entry) + "\n", + f"#{o + 1}", f"#{i + 1}") if edited else ""}) + + gone = sorted(set(range(len(old))) - {o for o in origs if o is not None}) + for o in gone: + before = old[o] if isinstance(old[o], dict) else {} + summary["removed"] += 1 + out.append({"kind": "removed", "label": driver_lint.label(before, o), "old_pos": o + 1, + "new_pos": None, "fields": [], + "diff": _udiff(_dump(before) + "\n", "", f"#{o + 1}", "(removed)")}) + out.sort(key=lambda c: (c["new_pos"] or c["old_pos"] or 0, c["kind"] != "removed")) + return out, summary + + +def _udiff(a, b, from_label, to_label): + # Whole entry as context: an entry is about 20 lines, and a reviewer + # wants to see which model a changed line belongs to. + return "".join(difflib.unified_diff(a.splitlines(keepends=True), b.splitlines(keepends=True), + fromfile=from_label, tofile=to_label, n=60)) diff --git a/webapp/services/hardware_coverage.py b/webapp/services/hardware_coverage.py index e4cdf6a..a58195d 100644 --- a/webapp/services/hardware_coverage.py +++ b/webapp/services/hardware_coverage.py @@ -33,10 +33,12 @@ from pathlib import PurePosixPath import config # Images whose HardwareDriver.json is checked, with the Windows they deploy. +# All three deploy Windows 11 now (engineer moved off Win10), so every one +# counts toward a model's gaps. IMAGES = ( ("gea-standard", "Win11"), ("gea-shopfloor", "Win11 LTSC"), - ("gea-engineer", "Win10"), + ("gea-engineer", "Win11"), ) KEEP = 10 # serials / versions remembered per model @@ -239,7 +241,7 @@ def coverage(include_ok=True): drivers = driver_status(model) bios = bios_status(model, rec.get("last_bios_version", "")) missing = [img for img, d in drivers.items() - if d["status"] in ("no-entry", "zip-missing") and img != "gea-engineer"] + if d["status"] in ("no-entry", "zip-missing")] gap = bool(missing) or bios["status"] in ("none", "file-missing") row = dict(rec, model=model, drivers=drivers, bios=bios, missing=missing, gap=gap, dismissed=model in state["dismissed"]) diff --git a/webapp/services/image_pages.py b/webapp/services/image_pages.py new file mode 100644 index 0000000..7f6a779 --- /dev/null +++ b/webapp/services/image_pages.py @@ -0,0 +1,348 @@ +"""Shared state for the per-image pages (GE Aerospace Images in the sidebar). + +Each image has one page with tabs, all drawn under templates/_image_header.html: + + tab id label endpoint URL + overview Overview images.overview /images/ + drivers Drivers images_drivers.page /images//drivers + unattend Unattend unattend_editor /images//unattend + os OS and Packages images.os_packages /images//os + +IMAGE_TABS is the single place that maps a tab to its endpoint: the header +partial, the sidebar and every redirect use it (tab_url). A page that moves +to another endpoint changes its row here and nothing else. + +header_info() is what the header partial shows; overview() feeds the +Overview tab. Both only read files, nothing here writes (saves go through +services/control_files.py). +""" + +import glob +import os +from pathlib import PurePosixPath + +from flask import url_for + +import config +from services import control_files, fs, hardware_coverage + +IMAGE_TABS = ( + {"id": "overview", "label": "Overview", "icon": "bi-clipboard-data", "endpoint": "images.overview"}, + {"id": "drivers", "label": "Drivers", "icon": "bi-hdd-network", "endpoint": "images_drivers.page"}, + {"id": "unattend", "label": "Unattend", "icon": "bi-file-earmark-code", "endpoint": "unattend_editor"}, + {"id": "os", "label": "OS and Packages", "icon": "bi-windows", "endpoint": "images.os_packages"}, +) +TAB_IDS = tuple(t["id"] for t in IMAGE_TABS) + +# PESetup loads the first of these that exists (docs/PESETUP-INTERNALS.md, +# "Paths it hardcodes"); the Unattend tab edits the fallback. +UNATTEND_PREFERRED = ("Deploy", "Tools", "GE", "XML", "FlatUnattendW10.xml") + +# osId values that mean Windows 10 in the GE catalogue: 18 is the only id +# the Dell_10 packs carry; 19, 20 and 21 appear only on Win11 packs (20 = +# Win11 LTSC 24H2, 21 = Win11 Pro 24H2). +WIN10_OS_IDS = {"18"} + + +def tab_url(image_type, tab="overview", **kw): + """URL of one tab of an image page.""" + endpoint = next(t["endpoint"] for t in IMAGE_TABS if t["id"] == tab) + return url_for(endpoint, image_type=image_type, **kw) + + +def tab_for_endpoint(endpoint): + """Tab id for a request endpoint, else None. Exact match first; then a + tab whose blueprint is its own (images_drivers.*) claims every endpoint + of that blueprint. The shared "images" blueprint only matches exactly.""" + for t in IMAGE_TABS: + if endpoint == t["endpoint"]: + return t["id"] + bp = (endpoint or "").partition(".")[0] + for t in IMAGE_TABS: + own = t["endpoint"].partition(".")[0] + if "." in t["endpoint"] and own == bp and own != "images": + return t["id"] + return None + + +# --- file locations ---------------------------------------------------------- + +def control_file(image_type, name): + return os.path.join(fs.control_path(image_type), name) + + +def user_selections_path(image_type): + return os.path.join(fs.tools_path(image_type), "user_selections.json") + + +def preferred_unattend_path(image_type): + return os.path.join(fs.image_root(image_type), *UNATTEND_PREFERRED) + + +# --- small readers ------------------------------------------------------------- + +def _list(data): + return data if isinstance(data, list) else [] + + +def _load(path): + """fs.load_json that also survives a file that is not UTF-8 (a UTF-16 + file saved by PowerShell 5.1 Out-File raises UnicodeDecodeError, which + fs.load_json lets through and which would fail every image tab).""" + try: + return fs.load_json(path) + except ValueError: + return [] + + +def drivers(image_type): + """HardwareDriver.json entries (the only driver file PESetup reads).""" + return _list(_load(control_file(image_type, "HardwareDriver.json"))) + + +def user_selections(image_type): + """(first object of user_selections.json, whole list) - the file is a + one-element JSON array written by Media Creator Lite.""" + raw = _load(user_selections_path(image_type)) + if isinstance(raw, list) and raw and isinstance(raw[0], dict): + return raw[0], raw + return {}, [] + + +def driver_os(entry): + """'Win11', 'Win10' or 'Other' for one driver entry. The pack file name + (win10_/win11_) decides first, then the Dell_10/Dell_11 folder, then + the osId list.""" + name = str(entry.get("fileName") or entry.get("FileName") or "").lower() + dest = str(entry.get("destinationDir") or entry.get("DestinationDir") or "").lower() + if name.startswith("win10"): + return "Win10" + if name.startswith("win11"): + return "Win11" + if "\\dell_10\\" in dest or "/dell_10/" in dest: + return "Win10" + if "\\dell_11\\" in dest or "/dell_11/" in dest: + return "Win11" + ids = [i.strip() for i in str(entry.get("osId") or "").split(",") if i.strip()] + if not ids: + return "Other" + return "Win10" if all(i in WIN10_OS_IDS for i in ids) else "Win11" + + +def zip_path(image_type, entry): + """Real path of an entry's zip (directories matched case-insensitively, + as on the live share), or None when it is not on disk.""" + rel = hardware_coverage._zip_relpath(entry) + if not rel: + return None + return _find_ci(fs.image_root(image_type), PurePosixPath(rel).parts) + + +def _find_ci(cur, parts): + """Depth-first case-insensitive lookup. Every case variant of a folder + is tried (Dell_11 can hold both Optiplex and OptiPlex as separate + folders), the exact spelling first.""" + if not parts: + return cur if os.path.isfile(cur) else None + try: + names = os.listdir(cur) + except OSError: + return None + hits = sorted((n for n in names if n.lower() == parts[0].lower()), key=lambda n: n != parts[0]) + for hit in hits: + found = _find_ci(os.path.join(cur, hit), parts[1:]) + if found: + return found + return None + + +def _stat(path): + try: + st = os.stat(path) + return {"exists": True, "size": st.st_size, "mtime": st.st_mtime} + except OSError: + return {"exists": False, "size": None, "mtime": None} + + +def operating_systems(image_type): + """OperatingSystem.json entries with the wim resolved on disk: + [{"entry", "osv", "wim_dir", "wim_path", "wim": _stat}].""" + out = [] + for entry in _list(_load(control_file(image_type, "OperatingSystem.json"))): + osv = entry.get("operatingSystemVersion") or {} + wim = osv.get("wim") or {} + dest = wim.get("DestinationDir") or wim.get("destinationDir") or "" + wim_dir = fs.resolve_destination(dest, image_type) + wim_path = os.path.join(wim_dir, "install.wim") if wim_dir else "" + out.append({"entry": entry, "osv": osv, "dest": dest, "wim_path": wim_path, + "wim": _stat(wim_path) if wim_path else _stat("")}) + return out + + +def packages(image_type): + """packages.json entries with the file resolved on disk: + [{"entry", "path", "file": _stat}].""" + out = [] + for entry in _list(_load(control_file(image_type, "packages.json"))): + name = entry.get("fileName") or entry.get("FileName") or "" + dest = entry.get("destinationDir") or entry.get("DestinationDir") or "" + folder = fs.resolve_destination(dest, image_type) + path = os.path.join(folder, name) if folder and name else "" + out.append({"entry": entry, "path": path, "file": _stat(path) if path else _stat("")}) + return out + + +def _current_os(rows, selection): + """The OS row imaging uses: the one user_selections.json selects, else + the first active one, else the first.""" + want = str(selection if selection is not None else "") + return (next((r for r in rows if str(r["osv"].get("id")) == want), None) + or next((r for r in rows if r["osv"].get("isActive")), None) + or (rows[0] if rows else None)) + + +def os_label(image_type): + """'Windows 11 Professional 24H2 (osId 21)' for the selected OS, else + the first active entry, else ''.""" + sel, _ = user_selections(image_type) + row = _current_os(operating_systems(image_type), sel.get("OperatingSystemSelection")) + if not row: + return "" + pick = row["osv"] + name = pick.get("productName") or pick.get("marketingName") or "Windows" + ver = pick.get("versionNumber") or "" + return f"{name} {ver} (osId {pick.get('id')})".replace(" ", " ") + + +# --- header + overview --------------------------------------------------------------- + +def header_info(image_type): + """What _image_header.html shows: name, OS, counts for the tab pills.""" + entries = drivers(image_type) + return { + "image_type": image_type, + "friendly_name": config.FRIENDLY_NAMES.get(image_type, image_type), + "os_label": os_label(image_type), + "driver_count": len(entries), + "package_count": len(packages(image_type)), + "has_unattend": os.path.isfile(fs.unattend_path(image_type)), + } + + +def _lint_counts(image_type): + """Findings of services/driver_lint (built by the Drivers tab) as + {"available", "total", "by_level"}; unavailable until that module + exists or when it fails.""" + try: + from services import driver_lint # noqa: WPS433 (optional module) + findings = driver_lint.lint_image(image_type) + except Exception: # missing module or a bad map: show "not available" + return {"available": False, "total": 0, "by_level": {}, "map_errors": 0} + by_level = {} + map_errors = 0 + for f in findings or []: + f = f if isinstance(f, dict) else vars(f) + level = str(f.get("level") or "INFO").upper() + by_level[level] = by_level.get(level, 0) + 1 + # A missing zip is an ERROR finding too, but the Overview counts + # those on their own tile; map_errors is what is wrong in the map. + if level in ("CRITICAL", "ERROR") and f.get("check") != "missing-zip": + map_errors += 1 + return {"available": True, "total": sum(by_level.values()), "by_level": by_level, + "map_errors": map_errors} + + +def _control_files(image_type): + """The files the image pages edit, with size/mtime and backups.""" + items = [ + ("HardwareDriver.json", control_file(image_type, "HardwareDriver.json"), "drivers", + "Driver map PESetup matches the PC model against"), + ("OperatingSystem.json", control_file(image_type, "OperatingSystem.json"), "os", + "Windows build, install.wim location and index"), + ("packages.json", control_file(image_type, "packages.json"), "os", + "Servicing packages applied offline (updates, .NET)"), + ("user_selections.json", user_selections_path(image_type), "os", + "Media Creator Lite selections (OS and driver families)"), + ("FlatUnattendW10.xml", fs.unattend_path(image_type), "unattend", + "Windows answer file (fallback location, the one this app edits)"), + ] + out = [] + for name, path, tab, what in items: + st = _stat(path) + backups = sorted(glob.glob(glob.escape(path) + ".bak-*")) + out.append(dict(st, name=name, path=path, + rel=os.path.relpath(path, fs.image_root(image_type)), + tab=tab, what=what, backups=len(backups))) + return out + + +def overview(image_type): + """Everything the Overview tab shows, from data already on the server.""" + entries = drivers(image_type) + by_os = {"Win11": 0, "Win10": 0, "Other": 0} + missing = [] + for i, e in enumerate(entries, 1): + by_os[driver_os(e)] += 1 + if not zip_path(image_type, e): + missing.append({"pos": i, "file": e.get("fileName") or e.get("FileName") or "", + "models": e.get("modelswminame") or e.get("models") or "", + "os": driver_os(e)}) + + sel, _ = user_selections(image_type) + oses = operating_systems(image_type) + pkgs = packages(image_type) + + # Hardware seen by WinPE with no working driver entry in THIS image. + gaps = [] + try: + for row in hardware_coverage.coverage(include_ok=False): + d = (row.get("drivers") or {}).get(image_type) + if d and d["status"] in ("no-entry", "zip-missing") and not row.get("dismissed"): + gaps.append({"model": row["model"], "status": d["status"], + "file": d.get("file", ""), "last_seen": row.get("last_seen"), + "skus": row.get("skus") or []}) + except Exception: + pass + covered = image_type in dict(hardware_coverage.IMAGES) + + # Newer Dell packs for files this map uses (last Updates > Dell scan). + dell = {"checked": None, "rows": []} + try: + from services import dell_updates + hints = dell_updates.update_hints()["drivers"] + dell["checked"] = (dell_updates.last_scan() or {}).get("checked") + seen = set() + for i, e in enumerate(entries, 1): + name = str(e.get("fileName") or e.get("FileName") or "") + if name.lower() in hints and name.lower() not in seen: + seen.add(name.lower()) + dell["rows"].append({"pos": i, "file": name, "ver": hints[name.lower()], + "models": e.get("modelswminame") or ""}) + except Exception: + pass + + preferred = preferred_unattend_path(image_type) + hw_extra = control_file(image_type, "hw_drivers.json") + families = {str(e.get("family") or "") for e in entries} + models = sel.get("HardwareModelSelection") or [] + return { + "drivers": {"total": len(entries), "by_os": by_os, "missing": missing}, + "lint": _lint_counts(image_type), + "os": {"selection": sel.get("OperatingSystemSelection"), "rows": oses, + "current": _current_os(oses, sel.get("OperatingSystemSelection"))}, + "packages": { + "total": len(pkgs), + "enabled": sum(1 for p in pkgs if p["entry"].get("enabled")), + "missing": [p for p in pkgs if p["entry"].get("enabled") and not p["file"]["exists"]], + }, + "models": {"total": len(models), + "no_entry": [m for m in models if str(m.get("Id") or "") not in families]}, + "files": _control_files(image_type), + "gaps": gaps, + "gaps_checked": covered, + "dell": dell, + "unattend_override": preferred if os.path.isfile(preferred) else None, + "hw_drivers_extra": hw_extra if os.path.isfile(hw_extra) else None, + "last_saves": {scope: control_files.last_save(image_type, scope) + for scope in ("drivers", "unattend", "os")}, + } diff --git a/webapp/services/ipxe_update.py b/webapp/services/ipxe_update.py new file mode 100644 index 0000000..bf9c5c1 --- /dev/null +++ b/webapp/services/ipxe_update.py @@ -0,0 +1,724 @@ +"""Keep the Secure Boot iPXE chain on the TFTP root current. + +What dnsmasq serves (see /etc/dnsmasq.conf): + * ipxe-shim.efi - Microsoft-signed shim (upstream x86_64-sb/shimx64.efi), + for the MACs tagged ipxe2; it always loads ipxe.efi from + the TFTP ROOT, whatever folder it came from; + * ipxe.efi - iPXE signed by the iPXE Secure Boot CA, loaded by the shim; + * ipxe-bcm.efi - the old 2021 Broadcom-signed build, everyone else. + Never touched here; + * ipxe.pxe - legacy BIOS iPXE (unsigned; BIOS has no Secure Boot). + +Upstream publishes the signed set as one tarball on every release: +https://github.com/ipxe/ipxe/releases/latest/download/ipxeboot.tar.gz, with +the signed x86 files in ipxeboot/x86_64-sb/. GitHub lists a sha256 digest per +release asset, which is checked before anything is unpacked, and every EFI +file's Authenticode chain is read with sbverify before it can be installed. +The chain check is by certificate name (sbverify --list); the cryptographic +trust comes from the GitHub digest. Do not swap in `sbverify --cert`: it +prints "Signature verification OK" even for a missing or unrelated cert. + +Why it matters: the shim and the Broadcom build are signed by Microsoft's +UEFI CA 2011, whose signing period ended in 2026. Machines that only trust +the 2023 CA, or a revocation (DBX/SBAT) update, will refuse them; the fix is +a newer upstream release, installed here. + +Staging runs as a background job (python3 -m services.ipxe_update stage); +install and rollback are a few MB of copying and run inline. +""" +from __future__ import annotations + +import fcntl +import glob +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import tarfile +import tempfile +from datetime import datetime + +import config +from services import updates_common as uc + +GITHUB_API = "https://api.github.com/repos/ipxe/ipxe/releases/latest" +RELEASES_PAGE = "https://github.com/ipxe/ipxe/releases" +LATEST_TARBALL_URL = "https://github.com/ipxe/ipxe/releases/latest/download/ipxeboot.tar.gz" +TARBALL = "ipxeboot.tar.gz" +JOB_UNIT = "pxe-webapp-ipxe-stage" + +# TFTP name -> path inside the tarball, and whether it must carry a +# Secure Boot signature (and from whom). +MANAGED = { + "ipxe-shim.efi": ("ipxeboot/x86_64-sb/shimx64.efi", "microsoft"), + "ipxe.efi": ("ipxeboot/x86_64-sb/ipxe.efi", "ipxe"), + "ipxe.pxe": ("ipxeboot/x86_64/ipxe.pxe", None), +} +# Installed by default; ipxe.pxe (legacy BIOS) only when asked. +DEFAULT_INSTALL = ("ipxe-shim.efi", "ipxe.efi") +# Shown on the page for reference, never written. +READ_ONLY = ("ipxe-bcm.efi",) + +NEED_LABEL = {"microsoft": "a Microsoft UEFI CA", "ipxe": "the iPXE Secure Boot CA"} + +# Issuer CNs, as sbverify prints them. +MS_CA_2011 = "Microsoft Corporation UEFI CA 2011" +MS_CA_2023 = "Microsoft UEFI CA 2023" +IPXE_CA = "iPXE Secure Boot CA" + + +def _dir(): + return uc.state_dir("ipxe") + + +def _job_file(): + return os.path.join(_dir(), "job.json") + + +# --- reading binaries ------------------------------------------------------- + +def signers(path): + """[(subject CN, issuer CN), ...] from the Authenticode chain, [] if none. + Raises FileNotFoundError when sbverify is not installed.""" + r = subprocess.run(["sbverify", "--list", path], capture_output=True, text=True, timeout=60) + pairs, subject = [], None + for line in r.stdout.splitlines(): + line = line.strip() + m = re.match(r"- subject: .*?CN=([^/]+)", line) + if m: + subject = m.group(1).strip() + continue + m = re.match(r"issuer: .*?CN=([^/]+)", line) + if m and subject is not None: + pairs.append((subject, m.group(1).strip())) + subject = None + return pairs + + +def signature_kind(pairs): + """microsoft-2023 | microsoft-2011 | microsoft | ipxe | other | unsigned. + A file signed under both Microsoft CAs counts as 2023.""" + issuers = {i for _, i in pairs} + names = " ".join(s + " " + i for s, i in pairs) + if MS_CA_2023 in issuers: + return "microsoft-2023" + if MS_CA_2011 in issuers: + return "microsoft-2011" + if "Microsoft" in names: + return "microsoft" + if IPXE_CA in issuers: + return "ipxe" + return "unsigned" if not pairs else "other" + + +def _sbat(data): + """The SBAT CSV a shim or iPXE build carries, as {component: [fields]}.""" + i = data.find(b"sbat,1,SBAT Version,") + if i < 0: + return {} + text = data[i:i + 4096].split(b"\0", 1)[0].decode("ascii", errors="replace") + out = {} + for line in text.splitlines(): + fields = line.split(",") + if len(fields) >= 2 and fields[0] != "sbat": + out[fields[0]] = fields + return out + + +def version_of(path): + try: + with open(path, "rb") as f: + data = f.read() + except OSError: + return "" + # A shim says so in its build ident; the shim's SBAT also names iPXE + # (shim.ipxe), so test this first. + m = re.search(rb"\$Version: ([^ $]{1,30})", data) + if m: + return "shim " + m.group(1).decode(errors="replace") + sbat = _sbat(data) + if "ipxe" in sbat and len(sbat["ipxe"]) >= 5: + return "iPXE " + sbat["ipxe"][4] + m = re.search(rb"([0-9]+\.[0-9]+\.[0-9]+\+? ?\(g[0-9a-f]+\))", data) + if m: + return "iPXE " + m.group(1).decode(errors="replace") + return "" + + +def sbat_generations(path): + """'shim 4, shim.ipxe 1': the generation numbers a revocation compares.""" + try: + with open(path, "rb") as f: + sbat = _sbat(f.read()) + except OSError: + return "" + return ", ".join(f"{k} {v[1]}" for k, v in sbat.items()) + + +def vendor_cert_cn(path): + """Subject CN of the certificate a shim trusts for its second stage (the + PE .vendor_cert section), or None. iPXE's shim embeds the iPXE Secure + Boot CA, so ipxe.efi must chain to that name.""" + try: + with open(path, "rb") as f: + d = f.read() + pe = struct.unpack_from(" 2: + continue + if s.startswith("conf-file="): + out += _conf_lines(s.split("=", 1)[1].strip(), depth + 1, seen) + elif s.startswith("conf-dir="): + parts = s.split("=", 1)[1].split(",") + out += _conf_dir_lines(parts[0].strip(), [p.strip() for p in parts[1:]], depth, seen) + if depth == 0: + implicit = os.path.join(os.path.dirname(path), "dnsmasq.d") + if os.path.isdir(implicit): + out += _conf_dir_lines(implicit, list(DEBIAN_CONF_DIR_EXCLUDE), depth, seen) + return out + + +def dnsmasq_routing(): + """Which boot file each class of client gets, read from dnsmasq.conf. + Read-only: this module never edits dnsmasq.""" + tag_macs, boots, raw = {}, [], [] + for src, s in _conf_lines(config.DNSMASQ_CONF): + key, _, val = s.partition("=") + key = key.strip() + if key == "dhcp-host": + fields = [f.strip() for f in val.split(",")] + macs = [f.lower() for f in fields if MAC_RE.match(f)] + for f in fields: + if f.startswith("set:"): + tag_macs.setdefault(f[4:], []).extend(macs) + elif key == "dhcp-boot": + fields = [f.strip() for f in val.split(",")] + need = [f[4:] for f in fields if f.startswith("tag:") and not f.startswith("tag:!")] + exclude = [f[5:] for f in fields if f.startswith("tag:!")] + rest = [f for f in fields if not f.startswith("tag:")] + boots.append({"file": rest[0] if rest else "", "need": need, "exclude": exclude, + "line": s, "source": src}) + if key in ("dhcp-boot", "dhcp-host", "dhcp-match", "dhcp-userclass", "dhcp-vendorclass", + "tag-if", "enable-tftp", "tftp-root"): + if key != "dhcp-host" or "set:" in val: + raw.append(s) + for b in boots: + b["macs"] = sorted({m for t in b["need"] for m in tag_macs.get(t, [])}) + # A path like ipxe2/ipxe-shim.efi is another copy, not the file in + # the TFTP root that this page installs. + rel = b["file"].lstrip("/") + b["managed"] = rel in MANAGED + b["read_only"] = rel in READ_ONLY + b["other_copy"] = rel not in MANAGED and os.path.basename(rel) in MANAGED + return {"tags": tag_macs, "boots": boots, "lines": raw, + "shim_macs": sorted({m for b in boots if os.path.basename(b["file"]) == "ipxe-shim.efi" + for m in b["macs"]})} + + +# --- upstream --------------------------------------------------------------- + +def _version_tuple(text): + m = re.search(r"(\d+)\.(\d+)(?:\.(\d+))?", text or "") + return tuple(int(x or 0) for x in m.groups()) if m else None + + +def compare(tag, installed_version): + """'update' | 'current' | 'newer' (installed is ahead) | 'unknown'.""" + a, b = _version_tuple(tag), _version_tuple(installed_version) + if not a or not b: + return "unknown" + return "update" if a > b else "current" if a == b else "newer" + + +def check_latest(timeout=20): + """Latest GitHub release: tag, date, tarball URL and sha256. Cached in + latest.json so the page and the stage job agree on what to fetch.""" + try: + rel = json.loads(uc.fetch_text(GITHUB_API, timeout=timeout)) + except ValueError: + raise RuntimeError("GitHub returned something that is not JSON (proxy login page?)") + asset = next((a for a in rel.get("assets", []) if a.get("name") == TARBALL), None) + if not rel.get("tag_name"): + raise RuntimeError("GitHub answer has no release tag") + if not asset: + raise RuntimeError(f"release {rel.get('tag_name')} has no {TARBALL}") + digest = str(asset.get("digest") or "") + page = str(rel.get("html_url") or "") # shown as a link on the page + info = { + "tag": rel.get("tag_name"), "name": rel.get("name") or "", + "published": rel.get("published_at"), + "page": page if page.startswith("https://") else RELEASES_PAGE, + "url": asset.get("browser_download_url"), + "size": asset.get("size"), + "sha256": digest.split(":", 1)[1].lower() if digest.startswith("sha256:") else "", + "checked": uc.now(), + } + uc.write_json(os.path.join(_dir(), "latest.json"), info) + return info + + +def last_check(): + return uc.read_json(os.path.join(_dir(), "latest.json")) + + +def _safe_tag(tag): + tag = re.sub(r"[^\w.-]", "_", str(tag or "")) + if not tag or tag in (".", ".."): + raise ValueError("bad release tag") + return tag + + +def _resolve(members, name): + """Follow symlinks/hardlinks inside the archive to a regular member.""" + for _ in range(8): + m = members.get(name) + if m is None or m.isfile(): + return m + if m.issym(): + name = os.path.normpath(os.path.join(os.path.dirname(name), m.linkname)) + elif m.islnk(): + name = os.path.normpath(m.linkname) + else: + return None + return None + + +def _progress(**fields): + data = uc.read_json(_job_file(), {}) or {} + data.update(fields, updated=uc.now()) + uc.write_json(_job_file(), data) + + +def stage(info=None): + """Download the tarball, verify it, unpack the managed files to + staged// and check each one's signature. Returns the staged record.""" + info = info or last_check() or check_latest() + if not info.get("sha256"): + raise RuntimeError("GitHub did not publish a sha256 for this release; stage it by hand") + folder = os.path.join(_dir(), "staged", _safe_tag(info["tag"])) + os.makedirs(folder, exist_ok=True) + tar_path = os.path.join(folder, TARBALL) + _progress(step="download", tag=info["tag"], size=info.get("size"), tarball=tar_path) + if os.path.isfile(tar_path) and uc.file_hash(tar_path).lower() != info["sha256"].lower(): + # A partial is resumed. A wrong file at full size or more cannot be + # (curl fails with 416/36 on every retry), so start it over. + size = info.get("size") or 0 + if not size or os.path.getsize(tar_path) >= size: + os.remove(tar_path) + if not (os.path.isfile(tar_path) and uc.file_hash(tar_path).lower() == info["sha256"].lower()): + # Room for a slow cap: twice the time the cap allows, at least 10 min. + cap = float(uc.settings().get("max_mbps") or 0) + size = float(info.get("size") or 0) + timeout = max(600, int(2 * size / (cap * 1024 * 1024))) if cap and size else 600 + uc.download(info["url"], tar_path, expect=info["sha256"], timeout=timeout) + _progress(step="verify") + files = {} + with tarfile.open(tar_path) as tar: + members = {os.path.normpath(m.name[2:] if m.name.startswith("./") else m.name): m + for m in tar.getmembers()} + for name, (inner, need) in MANAGED.items(): + m = _resolve(members, inner) + if m is None: + files[name] = {"ok": False, "error": f"{inner} not in the tarball"} + continue + dest = os.path.join(folder, name) + tmp = dest + ".part" + with tar.extractfile(m) as src, open(tmp, "wb") as out: + shutil.copyfileobj(src, out) + os.replace(tmp, dest) + d = describe(dest) + if d["sig_error"]: + ok, error = False, d["sig_error"] + elif need and not d["kind"].startswith(need): + ok, error = False, f"expected a signature under {NEED_LABEL[need]}, found {d['kind']}" + elif not d["size"]: + ok, error = False, "empty file" + else: + ok, error = True, "" + files[name] = dict(d, ok=ok, error=error, source=inner) + # The shim from this release must trust the CA its ipxe.efi is signed by. + shim, ipxe = files.get("ipxe-shim.efi") or {}, files.get("ipxe.efi") or {} + if shim.get("ok") and ipxe.get("ok") and shim.get("trusts"): + if shim["trusts"] not in {i for _, i in ipxe["signers"]}: + ipxe.update(ok=False, error=f"not signed under {shim['trusts']}, which this shim trusts") + record = {"tag": info["tag"], "published": info.get("published"), "folder": folder, + "tarball_sha256": info["sha256"], "files": files, "staged": uc.now()} + uc.write_json(os.path.join(folder, "staged.json"), record) + return record + + +def staged(tag=None): + root = os.path.join(_dir(), "staged") + if not os.path.isdir(root): + return [] + out = [] + for t in sorted(os.listdir(root), reverse=True): + rec = uc.read_json(os.path.join(root, t, "staged.json")) + if not (isinstance(rec, dict) and rec.get("tag") and isinstance(rec.get("files"), dict) + and rec["files"]): + continue # a hand-edited or half-written record has nothing to install + if tag is None or rec.get("tag") == tag: + # Where it is now, not the path recorded at staging time. + rec["folder"] = os.path.join(root, t) + out.append(rec) + out.sort(key=lambda r: r.get("staged") or "", reverse=True) + return out + + +# --- background stage job --------------------------------------------------- + +def job_state(): + """Last stage job's progress, with 'running' from systemd and, while + downloading, a percent from the partial file size.""" + data = uc.read_json(_job_file(), {}) or {} + try: + running = uc.job_running(JOB_UNIT) + except OSError: + running = False + data["running"] = running + if data.get("state") in ("queued", "running") and not running: + # Give systemd-run a moment to bring the unit up before calling it dead. + try: + age = datetime.now().timestamp() - os.path.getmtime(_job_file()) + except OSError: + age = 0 + if age > 20: + data["state"] = "interrupted" + if data.get("step") == "download" and data.get("size") and data.get("tarball"): + try: + done = os.path.getsize(data["tarball"]) + data["percent"] = min(100, int(100 * done / data["size"])) + except OSError: + data["percent"] = 0 + return data + + +def start_stage(): + """Queue the stage job (systemd-run). The page polls job_state().""" + if uc.job_running(JOB_UNIT): + raise RuntimeError("a download is already running") + info = last_check() + if not info: + raise RuntimeError("check for updates first") + uc.write_json(_job_file(), {"state": "queued", "tag": info["tag"], "started": uc.now(), + "updated": uc.now()}) + try: + uc.start_job(JOB_UNIT, ["services.ipxe_update", "stage"]) + except Exception as ex: + uc.write_json(_job_file(), {"state": "error", "tag": info["tag"], "error": str(ex), + "updated": uc.now()}) + raise + + +def run_stage_job(): + """Entry point of the background job. Never raises; the outcome goes to job.json.""" + _progress(state="running", step="start", error="", started=uc.now()) + try: + rec = stage(last_check()) + bad = [n for n, f in rec["files"].items() if not f.get("ok")] + _progress(state="done", step="done", tag=rec["tag"], finished=uc.now(), + failed=bad) + return 0 + except Exception as ex: # job boundary: record any failure for the page + _progress(state="error", error=str(ex)[:500], finished=uc.now()) + return 1 + + +# --- install / rollback ----------------------------------------------------- + +def _new_folder(root): + """A fresh timestamped folder; never reuses one (a rollback in the same + second as its backup must not overwrite the files it restores).""" + base = os.path.join(root, datetime.now().strftime("%Y%m%d-%H%M%S")) + folder, n = base, 1 + while True: + try: + os.makedirs(folder) + return folder + except FileExistsError: + folder, n = f"{base}-{n}", n + 1 + + +def _backup(names, reason): + folder = _new_folder(os.path.join(_dir(), "backups")) + saved, absent = [], [] + for name in names: + src = os.path.join(config.TFTP_ROOT, name) + if os.path.isfile(src): + shutil.copy2(src, os.path.join(folder, name)) + saved.append(name) + else: + absent.append(name) + uc.write_json(os.path.join(folder, "backup.json"), + {"reason": reason, "files": saved, "absent": absent, "at": uc.now()}) + return folder + + +class _Lock: + """One install or rollback at a time: two at once would back up each + other's half-placed files.""" + + def __enter__(self): + self.f = open(os.path.join(_dir(), ".install.lock"), "w") + try: + fcntl.flock(self.f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + self.f.close() + raise RuntimeError("another install or rollback is running; try again") from None + return self + + def __exit__(self, *exc): + self.f.close() + + +def _prepare(src, name, sha256=None): + """Copy src next to its TFTP name under a temporary name, fsynced, mode + 0644 and with the current file's owner. With sha256, the copy is checked. + Returns the temporary path; nothing is live yet.""" + dest = os.path.join(config.TFTP_ROOT, name) + fd, tmp = tempfile.mkstemp(dir=config.TFTP_ROOT, prefix=f".{name}.", suffix=".new") + try: + with open(src, "rb") as s, os.fdopen(fd, "wb") as out: + shutil.copyfileobj(s, out) + out.flush() + os.fsync(out.fileno()) + if sha256 and uc.file_hash(tmp).lower() != sha256.lower(): + raise RuntimeError(f"{name}: copy does not match its sha256, nothing replaced") + os.chmod(tmp, 0o644) + try: + st = os.stat(dest) + os.chown(tmp, st.st_uid, st.st_gid) + except OSError: + pass # no current file, or not root (local tests) + return tmp + except BaseException: + os.remove(tmp) + raise + + +def _place_all(items): + """[(src, name, sha256)] into the TFTP root. Every copy is made and + checked first, then each is renamed into place (atomic per file, so a + booting client never reads half a file). A failed copy leaves all the + live files as they were.""" + ready = [] + try: + for src, name, sha256 in items: + ready.append((_prepare(src, name, sha256), name)) + except BaseException: + for tmp, _ in ready: + os.remove(tmp) + raise + for tmp, name in ready: + os.replace(tmp, os.path.join(config.TFTP_ROOT, name)) + + +def install(tag, names=DEFAULT_INSTALL): + """Copy staged files for `tag` into the TFTP root after backing up the + current ones. Refuses a set with a failed signature check, and refuses to + install ipxe-shim.efi without its matching ipxe.efi (the shim only trusts + the iPXE CA the same release signs with). Returns the backup folder.""" + rec = next(iter(staged(tag)), None) + if not rec: + raise ValueError(f"{tag} is not staged") + names = [n for n in MANAGED if n in names] + if not names: + raise ValueError("nothing selected") + if "ipxe-shim.efi" in names and "ipxe.efi" not in names: + raise ValueError("install ipxe-shim.efi together with ipxe.efi") + if uc.job_running(JOB_UNIT): + raise RuntimeError("a download is running; wait for it to finish") + with _Lock(): + return _install(tag, rec, names) + + +def _install(tag, rec, names): + for n in names: + f = rec["files"].get(n) or {} + if not f.get("ok"): + raise ValueError(f"{n}: {f.get('error') or 'not staged'}") + path = os.path.join(rec["folder"], n) + if not os.path.isfile(path) or uc.file_hash(path) != f["sha256"]: + raise ValueError(f"{n}: staged copy changed since it was verified; download again") + if "ipxe.efi" in names and "ipxe-shim.efi" not in names: + # ipxe.efi alone must still be one the installed shim will start. + trusts = vendor_cert_cn(os.path.join(config.TFTP_ROOT, "ipxe-shim.efi")) + issuers = {i for _, i in rec["files"]["ipxe.efi"].get("signers", [])} + if trusts and trusts not in issuers: + raise ValueError(f"the installed shim trusts {trusts}; this ipxe.efi is not signed " + "under it, install the shim from the same release too") + backup = _backup(names, f"before installing {tag}") + _place_all([(os.path.join(rec["folder"], n), n, rec["files"][n]["sha256"]) for n in names]) + log = _installed_log() + for n in names: + log[n] = {"tag": tag, "sha256": rec["files"][n]["sha256"], "at": uc.now()} + uc.write_json(os.path.join(_dir(), "installed.json"), log) + return backup + + +def backups(): + root = os.path.join(_dir(), "backups") + if not os.path.isdir(root): + return [] + out = [] + for b in sorted(os.listdir(root), reverse=True): + folder = os.path.join(root, b) + if not os.path.isdir(folder): + continue + meta = uc.read_json(os.path.join(folder, "backup.json"), {}) + meta = meta if isinstance(meta, dict) else {} + files = [f for f in MANAGED if os.path.isfile(os.path.join(folder, f))] + out.append({"id": b, "reason": meta.get("reason", ""), "at": meta.get("at", ""), + "files": files, "absent": meta.get("absent", []), + "versions": {f: version_of(os.path.join(folder, f)) for f in files}}) + return out + + +def rollback(backup_id): + """Put the files of one backup back, after backing up what is there now.""" + bid = os.path.basename(str(backup_id or "")) + folder = os.path.join(_dir(), "backups", bid) + if not bid or bid.startswith(".") or not os.path.isdir(folder): + raise ValueError("no such backup") + files = [f for f in MANAGED if os.path.isfile(os.path.join(folder, f))] + if not files: + raise ValueError("that backup holds no boot files") + with _Lock(): + _backup(files, f"before rolling back to {bid}") + _place_all([(os.path.join(folder, f), f, uc.file_hash(os.path.join(folder, f))) + for f in files]) + log = _installed_log() + for f in files: + log.pop(f, None) + uc.write_json(os.path.join(_dir(), "installed.json"), log) + return files + + +if __name__ == "__main__": + if sys.argv[1:2] == ["stage"]: + sys.exit(run_stage_job()) + print("usage: python3 -m services.ipxe_update stage", file=sys.stderr) + sys.exit(2) diff --git a/webapp/services/mcl_ppkg.py b/webapp/services/mcl_ppkg.py new file mode 100644 index 0000000..e12b43c --- /dev/null +++ b/webapp/services/mcl_ppkg.py @@ -0,0 +1,371 @@ +"""MCL provisioning packages for gea-standard and gea-engineer. + +The vendor's bulk-enroll scripts in _shared/Applications/BPRT (every image's +Deploy/Applications/BPRT is a symlink to it) pick the package by filename: + + * the name must match ^((?:\\w|-)+?_){5,}v\\d+\\.\\d+.ppkg$ (PowerShell + -Match, so case-insensitive), e.g. GCCH_Prod_MCL-PERS_Exp_XXXXXXXX_v2.19.ppkg; + * Insert-BPRTToPPKG.ps1 then keeps the names that are -like + "__*" (a case-insensitive startswith, with + no trailing underscore) and FAILS enrollment when more than one is left. + +So an upload replaces the package with the same first three fields, and the +old one moves to BPRT-archive// - outside BPRT, because PESetup +copies the whole folder to every PC. Restore swaps an archived package back +the same way. Either is refused when the result would leave two packages that +one prefix claims (e.g. MCL-SH and MCL-SHX). + +A package is a WIM (Insert-BPRTToPPKG.ps1 renames it to .wim and mounts it), +so an upload must start with the WIM magic. The upload is written under the +archive folder first and renamed into BPRT only when complete, so a bay +copying BPRT mid-upload never sees a partial file. +""" +from __future__ import annotations + +import errno +import os +import re +import shutil +import threading +import time +from datetime import datetime + +import config + +# The vendor's own pattern (Insert-BPRTToPPKG.ps1 / Start-BulkEnrollOrchestrator.ps1). +VENDOR_PATTERN = re.compile(r"^((?:\w|-)+?_){5,}v\d+\.\d+.ppkg$", re.IGNORECASE) +_VERSION = re.compile(r"_(v\d+\.\d+)\.ppkg$", re.IGNORECASE) +# Insert-BPRTToPPKG.ps1 swaps this placeholder for the token's expiry date on +# the PC's copy; a name without it is a package already used on a PC. +_PLACEHOLDER = "_exp_xxxxxxxx_" +_BATCH = re.compile(r"^\d{8}-\d{6}(-\d+)?$") +WIM_MAGIC = b"MSWIM\0\0\0" + +# Serialises the archive-and-place step (uploads and restores). +_lock = threading.Lock() + + +def bprt_dir(): + return os.path.join(config.SHARED_DIR, "Applications", "BPRT") + + +def archive_dir(): + return os.path.join(config.SHARED_DIR, "Applications", "BPRT-archive") + + +def clean_name(name): + """Bare file name; some browsers send a full Windows path.""" + return os.path.basename(str(name or "").replace("\\", "/")).strip() + + +def is_mcl_name(name): + """True for a name the bulk-enroll scripts would treat as an MCL package + (the shopfloor SFLD names, e.g. GCCH_Prod_SFLD_v4.18.ppkg, are not).""" + return bool(VENDOR_PATTERN.match(clean_name(name))) + + +def parse(name, strict=True): + """{"prefix", "version"} for a usable package name, else ValueError. + The vendor regex ends in `.ppkg` with an unescaped dot, so it also takes + e.g. ..._v2.19-ppkg; strict (uploads, restores) wants the real extension, + while the listing (strict=False) shows what the enroll scripts would see.""" + if not VENDOR_PATTERN.match(name) or (strict and not name.lower().endswith(".ppkg")): + raise ValueError( + f"{name} does not match the vendor naming the enroll scripts need, " + "e.g. GCCH_Prod_MCL-PERS_Exp_XXXXXXXX_v2.19.ppkg") + m = _VERSION.search(name) + return {"prefix": "_".join(name.split("_")[:3]), + "version": m.group(1) if m else ""} + + +def _claims(prefix, name): + """What the vendor's `-like "$packagePrefix*"` does.""" + return name.lower().startswith(prefix.lower()) + + +def _clashes(names): + """{name: [other names its prefix also claims]} for a set of vendor-valid + names; empty when every prefix finds exactly one package.""" + out = {} + for n in names: + others = [m for m in names if m != n and _claims(parse(n, strict=False)["prefix"], m)] + if others: + out[n] = others + return out + + +def _check_unique(names, name): + """Refuse when `name` would be part of a clash: its prefix claims another + package, or another's prefix claims it. A clash among other packages + (made by hand) is flagged on the page but does not block this one.""" + clashes = {n: o for n, o in _clashes(names).items() if n == name or name in o} + if clashes: + n, others = next(iter(clashes.items())) + raise ValueError( + f"{parse(n, strict=False)['prefix']} would match {n} and {', '.join(others)}; " + "the enroll scripts fail when one prefix finds two packages") + + +def _pkg(folder, name): + st = os.stat(os.path.join(folder, name)) + info = {"filename": name, "size": st.st_size, "modified": st.st_mtime, + "placeholder": _PLACEHOLDER in name.lower()} + try: + info.update(parse(name, strict=False)) + except ValueError: + info.update(prefix="", version="") + return info + + +def _listed(folder, name): + return ((name.lower().endswith(".ppkg") or VENDOR_PATTERN.match(name)) + and os.path.isfile(os.path.join(folder, name))) + + +def packages(): + """Current packages in BPRT, sorted by prefix.""" + folder = bprt_dir() + out = [] + if os.path.isdir(folder): + for name in os.listdir(folder): + if _listed(folder, name): + out.append(_pkg(folder, name)) + out.sort(key=lambda p: (p["prefix"].lower(), p["filename"].lower())) + # A prefix that finds two packages breaks enrollment for that tenant/purpose. + clashes = _clashes([p["filename"] for p in out if p["prefix"]]) + for p in out: + p["conflict_with"] = clashes.get(p["filename"], []) + p["conflict"] = bool(p["conflict_with"]) + return out + + +def archived(): + """Archived packages, newest batch first: [{"batch", "filename", ...}].""" + root = archive_dir() + out = [] + if os.path.isdir(root): + for batch in sorted(os.listdir(root), reverse=True): + folder = os.path.join(root, batch) + if not _BATCH.match(batch) or not os.path.isdir(folder): + continue # .incoming (uploads in progress) and strays + for name in sorted(os.listdir(folder)): + if _listed(folder, name): + out.append(dict(_pkg(folder, name), batch=batch)) + return out + + +# --- moving files ----------------------------------------------------------- + +def _move(src, dst): + """Rename src onto dst. BPRT and its archive sit on one filesystem, so this + is a rename; the copy fallback only covers a share split across mounts.""" + try: + os.replace(src, dst) + except OSError as exc: + if exc.errno != errno.EXDEV: + raise + tmp = os.path.join(os.path.dirname(dst), f".{os.path.basename(dst)}.part") + try: + shutil.copy2(src, tmp) + os.replace(tmp, dst) + finally: + if os.path.exists(tmp): + os.remove(tmp) + os.remove(src) + + +def _new_batch(): + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + path = os.path.join(archive_dir(), stamp) + n = 1 + while os.path.exists(path): # two changes in one second must not share a batch + n += 1 + path = os.path.join(archive_dir(), f"{stamp}-{n}") + os.makedirs(path) + _match_perms(path, directory=True) + return path + + +def _archive(names): + """Move these BPRT packages into a new archive batch. Returns the batch + path (None when there was nothing to move).""" + if not names: + return None + batch = _new_batch() + done = [] + try: + for name in names: + _move(os.path.join(bprt_dir(), name), os.path.join(batch, name)) + done.append(name) + except OSError: + _unarchive(batch, done) + raise + return batch + + +def _unarchive(batch, names): + """Undo _archive after a failed placement: best effort, every file back.""" + for name in names: + try: + _move(os.path.join(batch, name), os.path.join(bprt_dir(), name)) + except OSError: + pass + try: + os.rmdir(batch) + except OSError: + pass + + +def _same_prefix(prefix): + """Valid BPRT package names whose own prefix equals this one.""" + return [p["filename"] for p in packages() + if p["prefix"] and p["prefix"].lower() == prefix.lower()] + + +def _plan(name, info): + """(names to archive) for placing `name`, or ValueError if the result + would leave a prefix finding two packages.""" + current = [p["filename"] for p in packages() if p["prefix"]] + replaced = _same_prefix(info["prefix"]) + _check_unique([n for n in current if n not in replaced] + [name], name) + return replaced + + +def _place(src, name, info): + """Archive what `name` replaces, then rename src into BPRT. Returns the + archived names.""" + with _lock: + replaced = _plan(name, info) + batch = _archive(replaced) + dest = os.path.join(bprt_dir(), name) + try: + _move(src, dest) + except OSError: + if batch: + _unarchive(batch, replaced) + raise + return replaced + + +# --- upload and restore ----------------------------------------------------- + +def check_upload(name): + """Validate a name before the body is saved: (clean name, info).""" + name = clean_name(name) + info = parse(name) + _plan(name, info) + return name, info + + +def install_upload(fileobj, name): + """Save an uploaded package into BPRT, archiving the one it replaces. + Returns (info, archived_names); info["filename"] is the name used and + info["warnings"] lists things worth telling the user.""" + name, info = check_upload(name) + os.makedirs(bprt_dir(), exist_ok=True) + incoming = os.path.join(archive_dir(), ".incoming") + os.makedirs(incoming, exist_ok=True) + _sweep(incoming) + _match_perms(archive_dir(), directory=True) + _match_perms(incoming, directory=True) + tmp = os.path.join(incoming, f"{os.getpid()}-{threading.get_ident()}-{name}.part") + try: + fileobj.save(tmp) + if os.path.getsize(tmp) == 0: + raise ValueError("uploaded file is empty") + with open(tmp, "rb") as f: + if f.read(len(WIM_MAGIC)) != WIM_MAGIC: + raise ValueError(f"{name} is not a provisioning package (no WIM header)") + moved = _place(tmp, name, info) + _match_perms(os.path.join(bprt_dir(), name)) + finally: + if os.path.exists(tmp): + os.remove(tmp) + info["filename"] = name + info["warnings"] = [] + if _PLACEHOLDER not in name.lower(): + info["warnings"].append( + f"{name} has no Exp_XXXXXXXX placeholder; a package already used on a PC " + "carries its BPRT and the enroll script cannot inject a new one") + return info, moved + + +def _sweep(incoming, age=86400): + """Drop temp files a crashed upload left behind (300 MB each, invisible + on the page). A live upload's file is seconds old.""" + cutoff = time.time() - age + for n in os.listdir(incoming): + path = os.path.join(incoming, n) + try: + if n.endswith(".part") and os.path.isfile(path) and os.path.getmtime(path) < cutoff: + os.remove(path) + except OSError: + pass + + +def archive(name): + """Move one package out of BPRT into a new archive batch (clears a + duplicate made by hand). Returns the bare name.""" + name = clean_name(name) + with _lock: + if name not in [p["filename"] for p in packages()]: + raise ValueError(f"{name or '(blank)'} is not a package in BPRT") + _archive([name]) + return name + + +def restore(batch, name): + """Put an archived package back, archiving whatever holds its prefix now.""" + batch = str(batch or "") + if not _BATCH.match(batch): + raise ValueError(f"{batch or '(blank)'} is not an archive batch") + name = clean_name(name) + folder = os.path.join(archive_dir(), batch) + src = os.path.join(folder, name) + if not name or not os.path.isfile(src): + raise ValueError(f"{batch}/{name} is not in the archive") + info = parse(name) + moved = _place(src, name, info) + try: + os.rmdir(folder) + except OSError: + pass # batch still holds other packages + info["filename"] = name + return info, moved + + +def _match_perms(path, directory=False): + """Uploads run as root; keep the share's pxe ownership so the Samba side + can still manage the file.""" + try: + st = os.stat(bprt_dir()) + os.chown(path, st.st_uid, st.st_gid) + os.chmod(path, 0o755 if directory else 0o744) + except OSError: + pass + + +def image_links(): + """[{"image", "state", "target"}] for every image with a + Deploy/Applications folder: state "ok" when its BPRT is the link to the + shared folder, "copy" when it is a real folder (an image import rsync + replaces the link with one, and then that image's PCs never see a + package changed here), "elsewhere" when it links somewhere else or + dangles, "none" when the image has no BPRT.""" + shared = os.path.realpath(bprt_dir()) + out = [] + for image in config.IMAGE_TYPES: + apps = os.path.join(config.SAMBA_SHARE, image, "Deploy", "Applications") + if not os.path.isdir(apps): + continue + path = os.path.join(apps, "BPRT") + target = "" + if os.path.islink(path): + target = os.readlink(path) + state = "ok" if os.path.realpath(path) == shared and os.path.isdir(path) else "elsewhere" + elif os.path.isdir(path): + state = "copy" + else: + state = "none" + out.append({"image": image, "state": state, "target": target, "path": path}) + return out diff --git a/webapp/services/unattend.py b/webapp/services/unattend.py index 8308147..4a88fdf 100644 --- a/webapp/services/unattend.py +++ b/webapp/services/unattend.py @@ -1,14 +1,20 @@ -"""Unattend.xml parser, builder, and form-data extractor. +"""Unattend.xml parser, form-edit applier, linter and builder. Round-trips a Windows ``FlatUnattendW10.xml`` between the disk format and a flat dict the editor template can render. ``parse_unattend()`` -reads disk + returns a dict; ``build_unattend_xml()`` takes the dict -+ returns a string for writing back; ``extract_form_data()`` converts -a Flask ``request.form`` MultiDict into the dict format that -``build_unattend_xml()`` expects. +reads disk + returns a dict; ``apply_form()`` writes a form dict back +INTO the existing file text (see "Form edits" below: only what changed +is touched, everything else stays byte for byte); ``lint_text()`` is +the webapp copy of scripts/lint-unattend.py; ``extract_form_data()`` +converts a Flask ``request.form`` MultiDict into the form dict. +``build_unattend_xml()`` (a fresh file from a dict) is kept for callers +that want one, but no save path uses it any more: it dropped comments, +rewrote *arch* and re-flowed every attribute. """ +import html import os +import re from lxml import etree @@ -54,8 +60,11 @@ def _settings_pass(root, pass_name): return s -def parse_unattend(xml_path): - """Parse an unattend.xml and return a dict of editable data.""" +def _empty_data(defaults): + """The dict parse_unattend returns. OOBE values: with defaults (no file + yet) the settings a new answer file gets; for an existing file a + setting that is not in it is "" (not set), so the form shows what the + file holds and a save does not add settings nobody chose.""" data = { "driver_paths": [], "computer_name": "", @@ -92,29 +101,64 @@ def parse_unattend(xml_path): "oobe_timezone": "", "raw_xml": "", } + if not defaults: + data["oobe"] = {k: "" for k in data["oobe"]} + return data + +def _comment_text(c): + return " ".join((c.text or "").split()) + + +def _notes(el): + """Comments that belong to a list item: the ones directly above it + (up to the previous element) and the ones inside it. Shown read-only + on the form; they move with the item (see Form edits).""" + lead = [] + for sib in el.itersiblings(preceding=True): + if not isinstance(sib, etree._Comment): + break + lead.append(_comment_text(sib)) + lead.reverse() + inner = [_comment_text(c) for c in el.iter(etree.Comment)] + return {"note": [t for t in lead if t], "inner_note": [t for t in inner if t]} + + +def parse_unattend(xml_path): + """Parse an unattend.xml and return a dict of editable data.""" if not os.path.isfile(xml_path): + data = _empty_data(defaults=True) data["raw_xml"] = UNATTEND_TEMPLATE return data + # errors="replace": a stray non-UTF-8 byte must not take the page down; + # the save path refuses a form save on such a file (views/images_unattend). + with open(xml_path, "r", encoding="utf-8", errors="replace") as fh: + return parse_unattend_text(fh.read()) - with open(xml_path, "r", encoding="utf-8") as fh: - raw = fh.read() + +def parse_unattend_text(raw): + """parse_unattend for text already read (a BOM, if any, is ignored).""" + data = _empty_data(defaults=False) data["raw_xml"] = raw try: - root = etree.fromstring(raw.encode("utf-8")) + root = etree.fromstring(raw.lstrip("\ufeff").encode("utf-8")) except etree.XMLSyntaxError: return data ns = {"u": config.UNATTEND_NS} # --- offlineServicing: DriverPaths --- + # By local name: the shopfloor file declares xmlns="" on + # PathAndCredentials, which takes it (and Path) OUT of the unattend + # namespace. A namespaced match missed it, and the old form save then + # dropped W:\Drivers from that image. for dp_el in root.xpath( - "u:settings[@pass='offlineServicing']//u:PathAndCredentials/u:Path", + "u:settings[@pass='offlineServicing']//*[local-name()='PathAndCredentials']" + "/*[local-name()='Path']", namespaces=ns, ): - if dp_el.text: - data["driver_paths"].append(dp_el.text.strip()) + data["driver_paths"].append((dp_el.text or "").strip()) # --- specialize: Shell-Setup --- for comp in root.xpath("u:settings[@pass='specialize']/u:component", namespaces=ns): @@ -138,11 +182,11 @@ def parse_unattend(xml_path): order_el = cmd.find(qn("Order")) path_el = cmd.find(qn("Path")) desc_el = cmd.find(qn("Description")) - data["specialize_commands"].append({ + data["specialize_commands"].append(dict({ "order": order_el.text.strip() if order_el is not None and order_el.text else "", "path": path_el.text.strip() if path_el is not None and path_el.text else "", "description": desc_el.text.strip() if desc_el is not None and desc_el.text else "", - }) + }, **_notes(cmd))) # --- oobeSystem --- for comp in root.xpath("u:settings[@pass='oobeSystem']/u:component", namespaces=ns): @@ -222,11 +266,11 @@ def parse_unattend(xml_path): order_el = sync.find(qn("Order")) cl_el = sync.find(qn("CommandLine")) desc_el = sync.find(qn("Description")) - data["firstlogon_commands"].append({ + data["firstlogon_commands"].append(dict({ "order": order_el.text.strip() if order_el is not None and order_el.text else "", "commandline": cl_el.text.strip() if cl_el is not None and cl_el.text else "", "description": desc_el.text.strip() if desc_el is not None and desc_el.text else "", - }) + }, **_notes(sync))) tz_el = comp.find(qn("TimeZone")) if tz_el is not None and tz_el.text: @@ -418,7 +462,11 @@ def build_unattend_xml(form_data): def extract_form_data(form): - """Pull structured data from the submitted form.""" + """Pull structured data from the submitted form (the field names of the + old one-page editor). A group of fields that is not in the post at all + is left out of the result, so apply_form leaves that part of the file + alone: a post that carries only a few fields must not empty every list + it did not send.""" data = {} dp_list = form.getlist("driver_path[]") @@ -493,4 +541,960 @@ def extract_form_data(form): data["oobe_timezone"] = form.get("oobe_timezone", "") - return data + sent = set(form.keys()) + groups = { + "driver_paths": {"driver_path[]"}, + "computer_name": {"computer_name"}, + "registered_organization": {"registered_organization"}, + "registered_owner": {"registered_owner"}, + "time_zone": {"time_zone"}, + "specialize_commands": {"spec_cmd_path[]", "spec_cmd_desc[]"}, + "oobe": {f"oobe_{k}" for k in OOBE_KEYS}, + "firstlogon_commands": {"fl_cmd_commandline[]", "fl_cmd_desc[]"}, + "user_accounts": {k for k in sent if k.startswith("account_")}, + "autologon": {k for k in sent if k.startswith("autologon_")}, + "intl": {k for k in sent if k.startswith("intl_")}, + "oobe_timezone": {"oobe_timezone"}, + } + return {k: v for k, v in data.items() if groups[k] & sent} + + +# --------------------------------------------------------------------------- +# Form edits: apply a form dict to the EXISTING file text +# --------------------------------------------------------------------------- +# +# The live answer files are hand-maintained: comments explain commands, +# processorArchitecture is the "*arch*" token PESetup substitutes, attributes +# are laid out over several lines, the shopfloor file declares xmlns="" on +# PathAndCredentials. Re-serializing the document (lxml, or +# build_unattend_xml) changes all of that, so a save here splices text: +# +# * a value that did not change is not touched (compared with what +# parse_unattend_text reads from the current file); +# * a changed value replaces only the text between its tags; +# * list items (commands, driver paths, accounts) are cut out of the file +# as whole lines and put back in the new order. The comments directly +# above an item, and any inside it, move with it; when an item is +# removed, the comments above it stay and go to the next item; +# * Order (commands) and wcm:keyValue (driver paths) are renumbered as +# little as possible: a removed item leaves a gap, an added or moved one +# takes a free number between its neighbours, and only when there is no +# room do the numbers after it shift (see _numbering); +# * a new element copies its indentation from its siblings (tabs or +# spaces, the file's line endings), a new list item copies its +# sibling's start tag (wcm:action, xmlns=""). +# +# The result is then parsed again and compared with the form; any +# difference refuses the save (ApplyError) rather than writing something +# other than what the page showed. Missing keys in the form dict mean +# "unchanged", so the JSON API can send only what it edits. + +WCM_DECL = 'xmlns:wcm="%s"' % config.WCM_NS +COMMON_COMPONENT_ATTRS = ('publicKeyToken="31bf3856ad364e35" language="neutral" ' + 'versionScope="nonSxS"') +OOBE_KEYS = ("HideEULAPage", "HideOEMRegistrationScreen", "HideOnlineAccountScreens", + "HideWirelessSetupInOOBE", "HideLocalAccountScreen", "NetworkLocation", + "ProtectYourPC", "SkipUserOOBE", "SkipMachineOOBE") +INTL_KEYS = (("InputLocale", "input_locale"), ("SystemLocale", "system_locale"), + ("UILanguage", "ui_language"), ("UserLocale", "user_locale")) +SHELL_KEYS = (("ComputerName", "computer_name"), + ("RegisteredOrganization", "registered_organization"), + ("RegisteredOwner", "registered_owner"), ("TimeZone", "time_zone")) + + +class ApplyError(ValueError): + """The form cannot be applied to this file exactly; use Raw XML.""" + + +_TOKEN = re.compile( + r"" + r"|<\?.*?\?>" + r"|" + r"|]*>" + r"|[^\s>]+)\s*>" + r"|<(?P[^\s/>!?]+)(?P(?:\s+[^\s=/>]+\s*=\s*(?:\"[^\"]*\"|'[^']*'))*)" + r"\s*(?P/?)>", + re.S) + + +class _Node: + """One element or comment of the file, as character offsets.""" + __slots__ = ("kind", "tag", "qname", "start", "end", "cstart", "cend", + "attrs", "children", "parent") + + def __init__(self, kind, tag="", qname="", start=0, attrs="", parent=None): + self.kind, self.tag, self.qname = kind, tag, qname + self.start, self.end = start, None + self.cstart = self.cend = None # content range; None when + self.attrs, self.children, self.parent = attrs, [], parent + + def els(self, tag=None): + return [c for c in self.children if c.kind == "el" and (tag is None or c.tag == tag)] + + def first(self, *path): + node = self + for tag in path: + found = node.els(tag) + if not found: + return None + node = found[0] + return node + + def descendants(self, tag): + out = [] + for c in self.els(): + if c.tag == tag: + out.append(c) + out.extend(c.descendants(tag)) + return out + + def attr(self, name): + m = re.search(r"(?:^|\s)(?:[\w.-]+:)?%s\s*=\s*(\"([^\"]*)\"|'([^']*)')" % re.escape(name), + self.attrs) + if not m: + return None + return html.unescape(m.group(2) if m.group(2) is not None else m.group(3)) + + +def _scan(text): + """Element/comment tree of a well-formed document, as offsets.""" + doc = _Node("doc", start=0) + stack = [doc] + for m in _TOKEN.finditer(text): + tok = m.group(0) + if m.group("open"): + q = m.group("open") + n = _Node("el", q.split(":")[-1], q, m.start(), m.group("attrs") or "", stack[-1]) + stack[-1].children.append(n) + if m.group("selfclose"): + n.end = m.end() + else: + n.cstart = m.end() + stack.append(n) + elif m.group("close"): + n = stack.pop() + if n.kind != "el" or n.qname != m.group("close"): + raise ApplyError("could not follow the element structure of the file") + n.cend, n.end = m.start(), m.end() + elif tok.startswith("