diff --git a/scripts/Upload-Image.ps1 b/scripts/Upload-Image.ps1 index 0b5e625..7cb70d2 100644 --- a/scripts/Upload-Image.ps1 +++ b/scripts/Upload-Image.ps1 @@ -1,8 +1,25 @@ # -# Upload-Image.ps1 — Copy MCL cached image to the PXE server +# Upload-Image.ps1 - Copy the Media Creator Lite cache to the PXE server, +# then PROVE every file arrived intact. # -# Reads user_selections.json to upload only the selected OS, matching -# packages, and config files. Drivers are EXCLUDED by default. +# 1. Checks the MCL cache before sending anything: +# - exactly one -Production.tag at the cache root (MCL writes it only +# when a run finishes; none = an unfinished or interrupted cache) +# - no Control file newer than that tag (MCL still writing / crashed) +# - the OS in OperatingSystem.json is the one in user_selections.json +# - every Control / Tools JSON is non-empty and parses +# - every WIM, update and driver pack hashes to the SHA-256 MCL's own +# Control JSONs publish (MCL resumes downloads onto old files and then +# trusts any file of the right size, so a cache can hold junk) +# 2. Hashes every file it will send into upload-manifest.json. +# 3. Copies with robocopy as before (SMB share \\\image-upload). +# 4. POSTs the manifest to the PXE webapp, which hashes the files on the +# server's own disk (no Windows SMB cache in the way) and lists any file +# that is missing or differs. +# 5. Re-sends those files over HTTP (small files; the server writes a temp +# file, checks the hash, then renames it) or robocopy /J /IS (large +# files), and verifies again, up to -MaxRounds times. +# The webapp's Image Import only accepts an upload that passed step 4. # # Usage: # .\Upload-Image.ps1 (selected OS + packages, no drivers) @@ -10,19 +27,23 @@ # .\Upload-Image.ps1 -CachePath "D:\MCL\Cache" (custom cache location) # .\Upload-Image.ps1 -Server 172.16.9.1 (custom server IP) # -# After upload, use the PXE webapp (http://172.16.9.1:9009) to import -# the uploaded content into the desired image type. +# Exit codes: 0 verified, 1 failed (details printed), 2 MCL cache not usable. # param( [string]$CachePath = "C:\ProgramData\GEAerospace\MediaCreator\Cache", [string]$Server = "172.16.9.1", + [int]$WebPort = 9009, [string]$User = "pxe-upload", [string]$Pass = "pxe", + [int]$MaxRounds = 3, [switch]$IncludeDrivers ) +$ErrorActionPreference = "Stop" $Share = "\\$Server\image-upload" +$ApiBase = "http://${Server}:$WebPort" +$ExcludedDeployDirs = @("Operating Systems", "Out-of-box Drivers", "Packages") function Format-Size { param([long]$Bytes) @@ -33,7 +54,79 @@ function Format-Size { function Resolve-DestDir { param([string]$Dir) - return ($Dir -replace '^\*destinationdir\*\\?', '') + $d = $Dir -replace '^\*destinationdir\*\\?', '' + return (($d -replace '^[A-Za-z]:\\', '').Trim('\')) +} + +function Write-Step { + param([string]$Text) + Write-Host "" + Write-Host "========================================" -ForegroundColor Cyan + Write-Host $Text -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan +} + +function Stop-Cache { + # The cache itself is unusable: nothing was sent. + param([string[]]$Lines) + Write-Host "" + foreach ($l in $Lines) { Write-Host "ERROR: $l" -ForegroundColor Red } + Write-Host "Nothing was uploaded." -ForegroundColor Red + exit 2 +} + +function Read-JsonFile { + param([string]$Path) + return (Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json) +} + +# HTTP to the webapp with HttpWebRequest and NO proxy: the work PC's system +# proxy (Zscaler) must not see traffic to the isolated imaging LAN. +# Returns @{ Status = ; Body = }; never throws on HTTP errors. +function Invoke-Pxe { + param([string]$Method, [string]$Path, [byte[]]$Body, [string]$InFile, + [string]$ContentType = "application/json", [int]$TimeoutSec = 120) + $req = [System.Net.HttpWebRequest]::Create("$ApiBase$Path") + $req.Method = $Method + $req.Proxy = $null + $req.Timeout = $TimeoutSec * 1000 + $req.ReadWriteTimeout = $TimeoutSec * 1000 + $req.KeepAlive = $false + if ($Body -or $InFile) { + $req.ContentType = $ContentType + if ($InFile) { $req.ContentLength = (Get-Item -LiteralPath $InFile).Length } + else { $req.ContentLength = $Body.Length } + try { $rs = $req.GetRequestStream() } catch { return @{ Status = 0; Body = $_.Exception.Message } } + try { + if ($InFile) { + $fs = [System.IO.File]::OpenRead($InFile) + try { $fs.CopyTo($rs) } finally { $fs.Dispose() } + } else { + $rs.Write($Body, 0, $Body.Length) + } + } finally { $rs.Dispose() } + } + try { + $resp = $req.GetResponse() + } catch { + # A .NET method call wraps the WebException in a + # MethodInvocationException: look at the inner one. + $ex = $_.Exception + while ($ex -and -not ($ex -is [System.Net.WebException])) { $ex = $ex.InnerException } + if (-not $ex -or -not $ex.Response) { return @{ Status = 0; Body = $_.Exception.Message } } + $resp = $ex.Response + } + try { + $sr = New-Object System.IO.StreamReader($resp.GetResponseStream()) + $text = $sr.ReadToEnd() + $sr.Dispose() + return @{ Status = [int]$resp.StatusCode; Body = $text } + } finally { $resp.Close() } +} + +function Get-ErrorText { + param($Resp) + try { return ($Resp.Body | ConvertFrom-Json).error } catch { return $Resp.Body } } Write-Host "" @@ -43,17 +136,60 @@ Write-Host "========================================" -ForegroundColor Cyan Write-Host "" # --- Validate source paths --- +$CachePath = $CachePath.TrimEnd('\') $DeployPath = Join-Path $CachePath "Deploy" $ToolsPath = Join-Path (Split-Path $CachePath -Parent) "Tools" if (-not (Test-Path $ToolsPath -PathType Container)) { $ToolsPath = "C:\ProgramData\GEAerospace\MediaCreator\Tools" } $SourcesZip = Join-Path $CachePath "Boot\Sources.zip" +$ControlPath = Join-Path $DeployPath "Control" if (-not (Test-Path $DeployPath -PathType Container)) { Write-Host "ERROR: Deploy directory not found at $DeployPath" -ForegroundColor Red Write-Host " .\Upload-Image.ps1 -CachePath ""D:\Path\To\Cache""" -ForegroundColor Yellow - exit 1 + exit 2 +} + +# --- MCL finished? One tag at the cache root, written at the very end --- +$tags = @(Get-ChildItem -LiteralPath $CachePath -Filter "*.tag" -File -ErrorAction SilentlyContinue) +if ($tags.Count -eq 0) { + Stop-Cache @("No MCL completion tag (*.tag) in $CachePath.", + "MCL writes it only when a run finishes: the cache is incomplete or the run was interrupted.", + "Run Media Creator Lite again and let it finish.") +} +if ($tags.Count -gt 1) { + Stop-Cache (@("More than one MCL tag in ${CachePath}:") + ($tags | ForEach-Object { " $($_.Name)" }) + + @("MCL leaves exactly one. Run Media Creator Lite again.")) +} +$tag = $tags[0] +$tagBytes = [System.IO.File]::ReadAllBytes($tag.FullName) +if ($tagBytes.Length -lt 8) { Stop-Cache @("MCL tag $($tag.Name) is damaged ($($tagBytes.Length) bytes). Run MCL again.") } +# MCL writes DateTime.Now.Ticks (local time) into the tag (TagFactory). +try { $tagTime = New-Object DateTime ([BitConverter]::ToInt64($tagBytes, 0)) } +catch { Stop-Cache @("MCL tag $($tag.Name) holds no valid time. Run MCL again.") } +$tagOs = $tag.BaseName -replace '-(Production|QA)$', '' + +$late = @(Get-ChildItem -LiteralPath $ControlPath -File -ErrorAction SilentlyContinue | + Where-Object { $_.LastWriteTime -gt $tagTime.AddSeconds(5) }) +if ($late.Count -gt 0) { + Stop-Cache (@("These Control files are newer than the MCL finish tag ($tagTime):") + + ($late | ForEach-Object { " $($_.Name) $($_.LastWriteTime)" }) + + @("MCL was still writing or crashed. Run Media Creator Lite again and let it finish.")) +} + +# --- Every Control / Tools JSON must be non-empty and parse --- +$badJson = @() +foreach ($dir in @($ControlPath, $ToolsPath)) { + if (-not (Test-Path $dir -PathType Container)) { continue } + foreach ($j in Get-ChildItem -LiteralPath $dir -Filter "*.json" -File -Recurse) { + if ($j.Length -eq 0) { $badJson += "$($j.FullName) (0 bytes)"; continue } + try { $null = Read-JsonFile $j.FullName } catch { $badJson += "$($j.FullName) (not valid JSON)" } + } +} +if ($badJson.Count -gt 0) { + Stop-Cache (@("Broken JSON in the MCL cache:") + ($badJson | ForEach-Object { " $_" }) + + @("Delete the cache folder ($CachePath) and run Media Creator Lite again.")) } # --- Parse user_selections.json --- @@ -61,49 +197,60 @@ $SelectionsFile = Join-Path $ToolsPath "user_selections.json" if (-not (Test-Path $SelectionsFile)) { Write-Host "ERROR: user_selections.json not found at $SelectionsFile" -ForegroundColor Red Write-Host " Run Media Creator Lite first to create a configuration." -ForegroundColor Yellow - exit 1 + exit 2 } -$selections = (Get-Content $SelectionsFile -Raw | ConvertFrom-Json)[0] +$selections = (Read-JsonFile $SelectionsFile)[0] $selectedOsId = $selections.OperatingSystemSelection $selectedModelIds = @($selections.HardwareModelSelection | ForEach-Object { $_.Id } | Select-Object -Unique) # --- Parse control JSONs --- -$ControlPath = Join-Path $DeployPath "Control" - $osJsonFile = Join-Path $ControlPath "OperatingSystem.json" $driverJsonFile = Join-Path $ControlPath "HardwareDriver.json" $pkgJsonFile = Join-Path $ControlPath "packages.json" +$langJsonFile = Join-Path $ControlPath "LanguagePacks.json" if (-not (Test-Path $osJsonFile)) { Write-Host "ERROR: OperatingSystem.json not found in $ControlPath" -ForegroundColor Red - exit 1 + exit 2 } -$osJson = Get-Content $osJsonFile -Raw | ConvertFrom-Json -$driverJson = if (Test-Path $driverJsonFile) { Get-Content $driverJsonFile -Raw | ConvertFrom-Json } else { @() } -$pkgJson = if (Test-Path $pkgJsonFile) { Get-Content $pkgJsonFile -Raw | ConvertFrom-Json } else { @() } +$osJson = @(Read-JsonFile $osJsonFile) +$driverJson = if (Test-Path $driverJsonFile) { @(Read-JsonFile $driverJsonFile) } else { @() } +$pkgJson = if (Test-Path $pkgJsonFile) { @(Read-JsonFile $pkgJsonFile) } else { @() } +$langJson = if (Test-Path $langJsonFile) { @(Read-JsonFile $langJsonFile) } else { @() } + +# --- The cached OS must be the selected one, and the one the tag names --- +$matchedOs = @($osJson | Where-Object { $_.operatingSystemVersion.id -eq [int]$selectedOsId }) +if ($matchedOs.Count -eq 0) { + Stop-Cache @("OperatingSystem.json does not hold the OS selected in user_selections.json (ID $selectedOsId).", + "Run Media Creator Lite again.") +} +$osv = $matchedOs[0].operatingSystemVersion +if (-not $tagOs.StartsWith($osv.productName, [StringComparison]::OrdinalIgnoreCase)) { + Stop-Cache @("The MCL tag says '$tagOs' but OperatingSystem.json holds '$($osv.productName)'.", + "The cache is a mix of two runs. Run Media Creator Lite again.") +} +$isLtsc = $osv.productName -match 'LTSC' +$suggested = if ($isLtsc) { "gea-shopfloor" } else { "standard-or-engineer" } # --- Resolve selections to paths --- - -# OS: match OperatingSystemSelection ID to OperatingSystem.json entries -$matchedOs = @($osJson | Where-Object { $_.operatingSystemVersion.id -eq [int]$selectedOsId }) $osDirs = @() $osTotalSize = [long]0 foreach ($os in $matchedOs) { - $rel = Resolve-DestDir $os.operatingSystemVersion.wim.DestinationDir - $osDirs += $rel + $osDirs += Resolve-DestDir $os.operatingSystemVersion.wim.DestinationDir $osTotalSize += $os.operatingSystemVersion.wim.size } -# Packages: enabled + matching OS ID -$matchedPkgs = @($pkgJson | Where-Object { $_.aOsIds -contains $selectedOsId -and $_.enabled -eq 1 }) +# Packages: enabled + matching OS ID (osId is a comma string in the new schema) +function Get-OsIds { param($e) if ($e.aOsIds) { return @($e.aOsIds) } return @("$($e.osId)" -split ',' | ForEach-Object { $_.Trim() }) } +$matchedPkgs = @($pkgJson | Where-Object { (Get-OsIds $_) -contains "$selectedOsId" -and $_.enabled -eq 1 }) $pkgTotalSize = [long]0 foreach ($pkg in $matchedPkgs) { $pkgTotalSize += $pkg.size } # Drivers: match selected model IDs (family) + OS ID, deduplicate by path $allMatchingDrivers = @($driverJson | Where-Object { - $selectedModelIds -contains $_.family -and $_.aOsIds -contains $selectedOsId + $selectedModelIds -contains $_.family -and (Get-OsIds $_) -contains "$selectedOsId" }) $allDriverDirSet = [ordered]@{} foreach ($drv in $allMatchingDrivers) { @@ -123,122 +270,232 @@ if ($IncludeDrivers) { # --- Display upload plan --- Write-Host " Cache: $CachePath" +Write-Host " MCL run: $tagOs, finished $tagTime" -ForegroundColor Green Write-Host " Server: $Server" +Write-Host " Target: $(if ($isLtsc) { 'Shop Floor (LTSC)' } else { 'Standard or Engineer (pick it in the webapp)' })" Write-Host "" Write-Host " Upload Plan (from user_selections.json):" -ForegroundColor Cyan Write-Host " ------------------------------------------" - -if ($matchedOs.Count -gt 0) { - $osName = $matchedOs[0].operatingSystemVersion.marketingName - Write-Host " OS: $osName ($(Format-Size $osTotalSize))" -ForegroundColor Green -} else { - Write-Host " OS: No match for selection ID $selectedOsId" -ForegroundColor Red -} - +Write-Host " OS: $($osv.marketingName) $($osv.productName) ($(Format-Size $osTotalSize))" -ForegroundColor Green Write-Host " Packages: $($matchedPkgs.Count) update(s) ($(Format-Size $pkgTotalSize))" -ForegroundColor Green - if ($IncludeDrivers) { Write-Host " Drivers: $($driverDirs.Count) model(s) ($(Format-Size $driverTotalSize))" -ForegroundColor Green } else { Write-Host " Drivers: SKIPPED -- $allDriverCount available, use -IncludeDrivers" -ForegroundColor Yellow } - Write-Host " Control: Always included" -ForegroundColor Gray Write-Host " Tools: $(if (Test-Path $ToolsPath) { 'Yes' } else { 'Not found' })" -ForegroundColor $(if (Test-Path $ToolsPath) { "Gray" } else { "Yellow" }) Write-Host " Sources: $(if (Test-Path $SourcesZip) { 'Yes (from Boot\Sources.zip)' } else { 'Not found' })" -ForegroundColor $(if (Test-Path $SourcesZip) { "Gray" } else { "Yellow" }) Write-Host "" -# --- Connect to SMB share --- -Write-Host "Connecting to $Share ..." -ForegroundColor Gray -net use $Share /delete 2>$null | Out-Null +# --- Sources: extract now so it can be hashed with everything else --- +$TempExtract = $null +$TempSources = $null +if (Test-Path $SourcesZip) { + $TempExtract = Join-Path $env:TEMP "SourcesExtract" + Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue + Expand-Archive $SourcesZip -DestinationPath $TempExtract -Force + # Handle nested Sources folder (zip may contain Sources/ at root) + $TempSources = $TempExtract + if ((Test-Path (Join-Path $TempExtract "Sources")) -and -not (Test-Path (Join-Path $TempExtract "Diskpart"))) { + $TempSources = Join-Path $TempExtract "Sources" + } +} -$netResult = net use $Share /user:$User $Pass 2>&1 +# --- Everything that will be sent: local file + path on the share --- +$sendList = New-Object System.Collections.Generic.List[object] +function Add-Tree { + param([string]$LocalDir, [string]$RelPrefix, [string[]]$SkipDirs = @()) + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { return } + $base = (Get-Item -LiteralPath $LocalDir).FullName.TrimEnd('\') + foreach ($f in Get-ChildItem -LiteralPath $base -Recurse -File -Force) { + $rel = $f.FullName.Substring($base.Length + 1) + if ($SkipDirs.Count -gt 0) { + # robocopy /XD skips that folder name at any depth + $segs = $rel.Split('\') + $skip = $false + for ($i = 0; $i -lt $segs.Count - 1; $i++) { if ($SkipDirs -contains $segs[$i]) { $skip = $true; break } } + if ($skip) { continue } + } + $sendList.Add([pscustomobject]@{ Local = $f.FullName; Rel = (Join-Path $RelPrefix $rel); Size = $f.Length }) + } +} +Add-Tree $DeployPath "Deploy" $ExcludedDeployDirs +foreach ($d in $osDirs) { Add-Tree (Join-Path $CachePath $d) $d } +foreach ($pkg in $matchedPkgs) { + $pdir = Resolve-DestDir $pkg.destinationDir + $rel = if ($pdir) { Join-Path $pdir $pkg.fileName } else { $pkg.fileName } + $local = Join-Path $CachePath $rel + if (Test-Path -LiteralPath $local -PathType Leaf) { + # File name as robocopy will write it (disk case), not as the JSON spells it + $item = Get-Item -LiteralPath $local + $rel = Join-Path (Split-Path $rel -Parent) $item.Name + $sendList.Add([pscustomobject]@{ Local = $item.FullName; Rel = $rel; Size = $item.Length }) + } +} +foreach ($d in $driverDirs) { Add-Tree (Join-Path $CachePath $d) $d } +Add-Tree $ToolsPath "Tools" +if ($TempSources) { Add-Tree $TempSources "Sources" } + +# The same file can be reached twice (an OS folder outside the excluded +# Deploy dirs, nested driver folders, two packages naming one file); a +# duplicate path would make the server refuse the whole manifest. +$seenRel = @{} +$dedup = New-Object System.Collections.Generic.List[object] +foreach ($f in $sendList) { + $k = ($f.Rel -replace '\\', '/').ToLower() + if (-not $seenRel.ContainsKey($k)) { $seenRel[$k] = $true; $dedup.Add($f) } +} +$sendList = $dedup + +# --- Hash everything; check payload against MCL's published hashes --- +Write-Step "Hashing $($sendList.Count) file(s) ($(Format-Size (($sendList | Measure-Object Size -Sum).Sum)))" +$mclHash = @{} +function Add-Mcl { param($Dest, $Name, $Hash) + # Only a real SHA-256 (as the server checks): anything else is not ours to judge + if (-not ($Dest -and $Name -and ("$Hash" -match '^[0-9a-fA-F]{64}$'))) { return } + $dd = Resolve-DestDir $Dest + $key = if ($dd) { Join-Path $dd $Name } else { "$Name" } + $mclHash[$key.ToLower()] = "$Hash".ToLower() } +foreach ($e in $osJson) { + $w = $e.operatingSystemVersion.wim + $n = if ($w.fileName) { $w.fileName } else { "install.wim" } + Add-Mcl $w.destinationDir $n $w.hash +} +foreach ($e in @($driverJson) + @($pkgJson)) { if ($e) { Add-Mcl $e.destinationDir $e.fileName $e.hash } } +foreach ($e in $langJson) { foreach ($lf in @($e.files)) { if ($lf) { Add-Mcl $lf.destFolder $lf.fileName $lf.hash } } } + +$totalBytes = [long](($sendList | Measure-Object Size -Sum).Sum) +$doneBytes = [long]0 +$badCache = @() +$i = 0 +foreach ($f in $sendList) { + $i++ + $pct = if ($totalBytes -gt 0) { [int](100 * $doneBytes / $totalBytes) } else { 100 } + Write-Progress -Activity "Hashing" -Status "$i/$($sendList.Count) $($f.Rel)" -PercentComplete $pct + $f | Add-Member -NotePropertyName Sha256 -NotePropertyValue (Get-FileHash -LiteralPath $f.Local -Algorithm SHA256).Hash.ToLower() + $doneBytes += $f.Size + $want = $mclHash[$f.Rel.ToLower()] + if ($want -and $want -ne $f.Sha256) { $badCache += $f } +} +Write-Progress -Activity "Hashing" -Completed +if ($badCache.Count -gt 0) { + Write-Host "" + Write-Host "These cache files do not match the hash MCL publishes for them (damaged download):" -ForegroundColor Red + foreach ($b in $badCache) { Write-Host " $($b.Rel)" -ForegroundColor Red } + Write-Host "" + Write-Host "MCL re-downloads a MISSING file correctly, but trusts a damaged one of the right size." -ForegroundColor Yellow + $ans = Read-Host "Delete these $($badCache.Count) file(s) from the MCL cache so MCL downloads them again? (Y/N)" + if ($ans -match '^[Yy]') { + foreach ($b in $badCache) { Remove-Item -LiteralPath $b.Local -Force; Write-Host " deleted $($b.Local)" -ForegroundColor Gray } + Write-Host "Now run Media Creator Lite again, then this script." -ForegroundColor Yellow + } + if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } + Write-Host "Nothing was uploaded." -ForegroundColor Red + exit 2 +} +Write-Host " All $($sendList.Count) file(s) hashed; payload matches MCL's published hashes." -ForegroundColor Green + +$pcName = ($env:COMPUTERNAME -replace '[^A-Za-z0-9._-]', '') +$uploadId = "{0}-{1}" -f (Get-Date -Format "yyyyMMdd-HHmmss"), $pcName +if ($uploadId.Length -gt 80) { $uploadId = $uploadId.Substring(0, 80) } +$manifest = [ordered]@{ + manifest_version = 1 + upload_id = $uploadId + created = (Get-Date -Format "yyyy-MM-ddTHH:mm:sszzz") + source_pc = $pcName + mcl = [ordered]@{ + tag = $tag.Name + finished = $tagTime.ToString("yyyy-MM-ddTHH:mm:ss") + os_id = [int]$osv.id + os_name = "$($osv.productName)" + os_version = "$($osv.versionNumber)" + build = "$($osv.buildNumber)" + suggested_target = $suggested + } + files = @($sendList | ForEach-Object { [ordered]@{ path = ($_.Rel -replace '\\', '/'); size = [long]$_.Size; sha256 = $_.Sha256 } }) +} +$manifestBytes = [System.Text.Encoding]::UTF8.GetBytes(($manifest | ConvertTo-Json -Depth 6 -Compress)) + +# --- Is the webapp reachable? (before spending time on the copy) --- +$ping = Invoke-Pxe -Method GET -Path "/api/upload/verify" -TimeoutSec 15 +if ($ping.Status -ne 200) { + Write-Host "ERROR: the PXE webapp at $ApiBase does not answer ($($ping.Status) $($ping.Body))." -ForegroundColor Red + Write-Host " The upload cannot be verified without it. Check the server, then run this again." -ForegroundColor Yellow + if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } + exit 1 +} + +# --- Connect to SMB share --- +Write-Host "" +Write-Host "Connecting to $Share ..." -ForegroundColor Gray +# net use goes through cmd: in Windows PowerShell 5.1 a native command's +# redirected stderr becomes an error record, which ErrorAction Stop turns +# into a crash ("The network connection could not be found"). +cmd /c "net use ""$Share"" /delete /y >nul 2>&1" | Out-Null + +$netResult = cmd /c "net use ""$Share"" /user:$User ""$Pass"" 2>&1" if ($LASTEXITCODE -ne 0) { Write-Host "ERROR: Could not connect to $Share" -ForegroundColor Red Write-Host $netResult -ForegroundColor Red Write-Host "" Write-Host "Make sure:" -ForegroundColor Yellow Write-Host " - The PXE server is running at $Server" -ForegroundColor Yellow - Write-Host " - This PC is on the 10.9.100.x network" -ForegroundColor Yellow + Write-Host " - This PC is on the 172.16.9.x imaging network" -ForegroundColor Yellow Write-Host " - Samba is running on the PXE server" -ForegroundColor Yellow + if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } exit 1 } Write-Host "Connected." -ForegroundColor Green -Write-Host "" $failed = $false $stepNum = 0 -$totalSteps = 1 # Deploy base always -if ($matchedOs.Count -gt 0) { $totalSteps++ } +$totalSteps = 2 # Deploy base + OS always if ($matchedPkgs.Count -gt 0) { $totalSteps++ } if ($IncludeDrivers -and $driverDirs.Count -gt 0) { $totalSteps++ } if (Test-Path $ToolsPath -PathType Container) { $totalSteps++ } -if (Test-Path $SourcesZip) { $totalSteps++ } +if ($TempSources) { $totalSteps++ } + +# robocopy exit codes 8 and up are failures +$rcArgs = @("/R:3", "/W:5", "/NP", "/ETA") # --- Step: Deploy base (Control, Applications, config -- skip big dirs) --- $stepNum++ -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "[$stepNum/$totalSteps] Copying Deploy\ base (Control, Applications, config) ..." -ForegroundColor Cyan -Write-Host "========================================" -ForegroundColor Cyan - -robocopy $DeployPath "$Share\Deploy" /E /XD "Operating Systems" "Out-of-box Drivers" "Packages" /R:3 /W:5 /NP /ETA -if ($LASTEXITCODE -ge 8) { - Write-Host "ERROR: Deploy base copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red - $failed = $true -} +Write-Step "[$stepNum/$totalSteps] Copying Deploy\ base (Control, Applications, config) ..." +robocopy $DeployPath "$Share\Deploy" /E /XD @ExcludedDeployDirs @rcArgs +if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Deploy base copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } # --- Step: Operating System --- -if ($matchedOs.Count -gt 0) { - $stepNum++ - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "[$stepNum/$totalSteps] Copying Operating System ..." -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - - foreach ($osDir in $osDirs) { - $src = Join-Path $CachePath $osDir - $dst = Join-Path $Share $osDir - if (Test-Path $src -PathType Container) { - Write-Host " $osDir" -ForegroundColor Gray - robocopy $src $dst /E /R:3 /W:5 /NP /ETA - if ($LASTEXITCODE -ge 8) { - Write-Host "ERROR: OS copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red - $failed = $true - } - } else { - Write-Host " SKIPPED (not cached): $osDir" -ForegroundColor Yellow - } +$stepNum++ +Write-Step "[$stepNum/$totalSteps] Copying Operating System ..." +foreach ($osDir in $osDirs) { + $src = Join-Path $CachePath $osDir + if (Test-Path -LiteralPath $src -PathType Container) { + Write-Host " $osDir" -ForegroundColor Gray + robocopy $src (Join-Path $Share $osDir) /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: OS copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } + } else { + Write-Host " SKIPPED (not cached): $osDir" -ForegroundColor Yellow } } # --- Step: Packages --- if ($matchedPkgs.Count -gt 0) { $stepNum++ - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "[$stepNum/$totalSteps] Copying Packages ..." -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - - # Group packages by destination directory for efficient robocopy + Write-Step "[$stepNum/$totalSteps] Copying Packages ..." $pkgGroups = [ordered]@{} foreach ($pkg in $matchedPkgs) { $rel = Resolve-DestDir $pkg.destinationDir if (-not $pkgGroups.Contains($rel)) { $pkgGroups[$rel] = @() } $pkgGroups[$rel] += $pkg.fileName } - foreach ($dir in $pkgGroups.Keys) { $src = Join-Path $CachePath $dir - $dst = Join-Path $Share $dir - $files = $pkgGroups[$dir] - if (Test-Path $src -PathType Container) { - foreach ($f in $files) { Write-Host " $f" -ForegroundColor Gray } - $robocopyArgs = @($src, $dst) + $files + @("/R:3", "/W:5", "/NP", "/ETA") + if (Test-Path -LiteralPath $src -PathType Container) { + foreach ($f in $pkgGroups[$dir]) { Write-Host " $f" -ForegroundColor Gray } + $robocopyArgs = @($src, (Join-Path $Share $dir)) + $pkgGroups[$dir] + $rcArgs & robocopy @robocopyArgs - if ($LASTEXITCODE -ge 8) { - Write-Host "ERROR: Package copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red - $failed = $true - } + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Package copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } } else { Write-Host " SKIPPED (not cached): $dir" -ForegroundColor Yellow } @@ -248,23 +505,15 @@ if ($matchedPkgs.Count -gt 0) { # --- Step: Drivers (only with -IncludeDrivers) --- if ($IncludeDrivers -and $driverDirs.Count -gt 0) { $stepNum++ - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "[$stepNum/$totalSteps] Copying Drivers ($($driverDirs.Count) model(s)) ..." -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - + Write-Step "[$stepNum/$totalSteps] Copying Drivers ($($driverDirs.Count) model(s)) ..." $drvCopied = 0 foreach ($drvDir in $driverDirs) { $drvCopied++ $src = Join-Path $CachePath $drvDir - $dst = Join-Path $Share $drvDir - if (Test-Path $src -PathType Container) { + if (Test-Path -LiteralPath $src -PathType Container) { Write-Host " [$drvCopied/$($driverDirs.Count)] $drvDir" -ForegroundColor Gray - robocopy $src $dst /E /R:3 /W:5 /NP /ETA - if ($LASTEXITCODE -ge 8) { - Write-Host "ERROR: Driver copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red - $failed = $true - } + robocopy $src (Join-Path $Share $drvDir) /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Driver copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } } else { Write-Host " SKIPPED (not cached): $drvDir" -ForegroundColor Yellow } @@ -274,106 +523,118 @@ if ($IncludeDrivers -and $driverDirs.Count -gt 0) { # --- Step: Tools --- if (Test-Path $ToolsPath -PathType Container) { $stepNum++ - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "[$stepNum/$totalSteps] Copying Tools\ ..." -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - - robocopy $ToolsPath "$Share\Tools" /E /R:3 /W:5 /NP /ETA - if ($LASTEXITCODE -ge 8) { - Write-Host "ERROR: Tools copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red - $failed = $true - } + Write-Step "[$stepNum/$totalSteps] Copying Tools\ ..." + robocopy $ToolsPath "$Share\Tools" /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Tools copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } } # --- Step: Sources --- -$TempSources = $null -if (Test-Path $SourcesZip) { +if ($TempSources) { $stepNum++ - Write-Host "" - Write-Host "========================================" -ForegroundColor Cyan - Write-Host "[$stepNum/$totalSteps] Extracting and copying Sources\ ..." -ForegroundColor Cyan - Write-Host "========================================" -ForegroundColor Cyan - - $TempExtract = Join-Path $env:TEMP "SourcesExtract" - Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue - Expand-Archive $SourcesZip -DestinationPath $TempExtract -Force - - # Handle nested Sources folder (zip may contain Sources/ at root) - $TempSources = $TempExtract - if ((Test-Path (Join-Path $TempExtract "Sources")) -and -not (Test-Path (Join-Path $TempExtract "Diskpart"))) { - $TempSources = Join-Path $TempExtract "Sources" - } - - robocopy $TempSources "$Share\Sources" /E /R:3 /W:5 /NP /ETA - if ($LASTEXITCODE -ge 8) { - Write-Host "ERROR: Sources copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red - $failed = $true - } - - Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue + Write-Step "[$stepNum/$totalSteps] Copying Sources\ ..." + robocopy $TempSources "$Share\Sources" /E @rcArgs + if ($LASTEXITCODE -ge 8) { Write-Host "ERROR: Sources copy failed (exit code $LASTEXITCODE)" -ForegroundColor Red; $failed = $true } } -# --- Verify small files (SMB write-cache workaround) --- -Write-Host "" -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "Verifying small files ..." -ForegroundColor Cyan -Write-Host "========================================" -ForegroundColor Cyan +# --- Verify on the server; re-send what is wrong --- +$byRel = @{} +foreach ($f in $sendList) { $byRel[($f.Rel -replace '\\', '/').ToLower()] = $f } +$verified = $false +$result = $null +for ($round = 1; $round -le $MaxRounds + 1; $round++) { + Write-Step "Verifying on the server (round $round) ..." + $r = Invoke-Pxe -Method POST -Path "/api/upload/verify" -Body $manifestBytes -TimeoutSec 120 + if ($r.Status -ne 202) { + Write-Host "ERROR: verify request refused ($($r.Status)): $(Get-ErrorText $r)" -ForegroundColor Red + break + } + $deadline = (Get-Date).AddHours(2) + $postedAt = Get-Date + do { + if ((Get-Date) -gt $deadline) { $result = $null; break } + if ($result -and $result.upload_id -ne $uploadId -and ((Get-Date) - $postedAt).TotalSeconds -gt 120) { + Write-Host "ERROR: the server is working on another upload ($($result.upload_id)). Run this again later." -ForegroundColor Red + $result = $null; break + } + Start-Sleep -Seconds 2 + $s = Invoke-Pxe -Method GET -Path "/api/upload/verify" -TimeoutSec 30 + $result = $null + if ($s.Status -eq 200) { $result = $s.Body | ConvertFrom-Json } + if ($result -and $result.state -eq "running" -and $result.total_bytes -gt 0) { + Write-Progress -Activity "Server is hashing the upload" -Status "$($result.done_files)/$($result.total_files) files" ` + -PercentComplete ([Math]::Min(100, [int](100 * $result.done_bytes / $result.total_bytes))) + } + } while (-not $result -or $result.state -eq "running" -or $result.upload_id -ne $uploadId) + Write-Progress -Activity "Server is hashing the upload" -Completed -$fixCount = 0 -$verifyPairs = @( - @{ Local = (Join-Path $DeployPath "Control"); Remote = "$Share\Deploy\Control" } -) -if (Test-Path $ToolsPath -PathType Container) { - $verifyPairs += @{ Local = $ToolsPath; Remote = "$Share\Tools" } -} + if (-not $result) { Write-Host "ERROR: the server did not finish verifying within 2 hours." -ForegroundColor Red; break } + if ($result.state -eq "error") { Write-Host "ERROR: the server could not verify: $($result.error)" -ForegroundColor Red; break } + if ($result.ok) { $verified = $true; break } -foreach ($pair in $verifyPairs) { - $localDir = $pair.Local - $remoteDir = $pair.Remote - if (-not (Test-Path $localDir -PathType Container)) { continue } + foreach ($b in @($result.bad_in_cache)) { if ($b) { Write-Host " BAD IN MCL CACHE: $($b.path) ($($b.reason))" -ForegroundColor Red } } + if (@($result.bad_in_cache).Count -gt 0) { break } # re-sending a bad source cannot help - Get-ChildItem -Path $localDir -Recurse -File -ErrorAction SilentlyContinue | - Where-Object { $_.Length -gt 0 -and $_.Length -lt 1MB } | - ForEach-Object { - $rel = $_.FullName.Substring($localDir.Length) - $dstFile = Join-Path $remoteDir $rel - if (Test-Path $dstFile) { - $dstSize = (Get-Item $dstFile).Length - if ($dstSize -ne $_.Length) { - Write-Host " Fixing: $rel ($dstSize -> $($_.Length) bytes)" -ForegroundColor Yellow - $bytes = [System.IO.File]::ReadAllBytes($_.FullName) - [System.IO.File]::WriteAllBytes($dstFile, $bytes) - $fixCount++ - } + $redo = @(@($result.missing) + @($result.mismatch | ForEach-Object { $_.path }) | Where-Object { $_ }) + Write-Host " $($redo.Count) file(s) missing or different on the server." -ForegroundColor Yellow + if ($round -gt $MaxRounds) { break } + foreach ($rel in $redo) { + $f = $byRel[$rel.ToLower()] + if (-not $f) { continue } + if ($f.Size -le [long]$result.small_max) { + $q = "?upload_id=$([uri]::EscapeDataString($uploadId))&path=$([uri]::EscapeDataString($rel))" + $p = Invoke-Pxe -Method PUT -Path "/api/upload/file$q" -InFile $f.Local -ContentType "application/octet-stream" -TimeoutSec 300 + if ($p.Status -eq 200) { Write-Host " re-sent over HTTP: $rel" -ForegroundColor Green } + else { Write-Host " HTTP re-send failed ($($p.Status)) for ${rel}: $(Get-ErrorText $p)" -ForegroundColor Red } + } else { + $dstDir = Join-Path $Share (Split-Path ($f.Rel) -Parent) + $dstFile = Join-Path $dstDir (Split-Path $f.Local -Leaf) + Write-Host " re-copying (delete, then robocopy /J): $rel" -ForegroundColor Gray + # robocopy calls a same-size, same-time file "same" or "modified" + # and skips it, even with /IS /IT. Delete the bad copy so it is a + # new file; /IS /IT /IM stay as a backstop if the delete fails. + Remove-Item -LiteralPath $dstFile -Force -ErrorAction SilentlyContinue + $rc = robocopy (Split-Path $f.Local -Parent) $dstDir (Split-Path $f.Local -Leaf) /J /IS /IT /IM @rcArgs + if ($LASTEXITCODE -ge 8 -or $LASTEXITCODE -eq 0) { + # 0 = nothing copied: robocopy skipped it again + Write-Host " robocopy did not copy $rel (exit $LASTEXITCODE):" -ForegroundColor Red + $rc | Where-Object { $_ -match '\S' } | Select-Object -Last 8 | ForEach-Object { Write-Host " $_" } } } + } } -if ($fixCount -eq 0) { - Write-Host " All files verified OK." -ForegroundColor Green -} else { - Write-Host " Fixed $fixCount file(s)." -ForegroundColor Yellow -} - -# --- Disconnect --- -net use $Share /delete 2>$null | Out-Null +# --- Disconnect and clean up --- +cmd /c "net use ""$Share"" /delete /y >nul 2>&1" | Out-Null +if ($TempExtract) { Remove-Item -Recurse -Force $TempExtract -ErrorAction SilentlyContinue } # --- Summary --- Write-Host "" -if ($failed) { - Write-Host "========================================" -ForegroundColor Red - Write-Host " Upload completed with errors." -ForegroundColor Red - Write-Host "========================================" -ForegroundColor Red -} else { - Write-Host "========================================" -ForegroundColor Green - Write-Host " Upload complete!" -ForegroundColor Green - Write-Host "========================================" -ForegroundColor Green +if ($failed -and $verified) { + Write-Host " (robocopy reported errors on the first pass; the files it missed were re-sent and verified.)" -ForegroundColor Yellow } -Write-Host "" -Write-Host "Next steps:" -ForegroundColor Cyan -Write-Host " 1. Open the PXE webapp: http://$Server`:9009" -ForegroundColor White -Write-Host " 2. Go to Image Import" -ForegroundColor White -Write-Host " 3. Select source 'image-upload' and target image type" -ForegroundColor White -Write-Host " 4. Click Import" -ForegroundColor White -Write-Host "" +if ($verified) { + Write-Host "========================================" -ForegroundColor Green + Write-Host " Upload complete: all $($sendList.Count) file(s) verified on the server." -ForegroundColor Green + Write-Host "========================================" -ForegroundColor Green + if ([int]$result.extra_count -gt 0) { + Write-Host "" + Write-Host " NOTE: the upload folder also holds $($result.extra_count) file(s) from an earlier upload." -ForegroundColor Yellow + Write-Host " Import is blocked until you remove them (button on the Image Import page)." -ForegroundColor Yellow + } + Write-Host "" + Write-Host "Next steps:" -ForegroundColor Cyan + Write-Host " 1. Open the PXE webapp: http://${Server}:$WebPort/images/import" -ForegroundColor White + Write-Host " 2. Target: $(if ($isLtsc) { 'Shop Floor (the only choice for LTSC)' } else { 'Standard or Engineer' })" -ForegroundColor White + Write-Host " 3. Click Start import" -ForegroundColor White + Write-Host "" + exit 0 +} +Write-Host "========================================" -ForegroundColor Red +Write-Host " Upload NOT verified." -ForegroundColor Red +Write-Host "========================================" -ForegroundColor Red +if ($result) { + foreach ($m in @($result.missing)) { if ($m) { Write-Host " missing: $m" -ForegroundColor Red } } + foreach ($m in @($result.mismatch)) { if ($m) { Write-Host " differs: $($m.path) ($($m.reason))" -ForegroundColor Red } } +} +Write-Host "The webapp will not import this upload. Run the script again." -ForegroundColor Yellow +exit 1 diff --git a/webapp/app.py b/webapp/app.py index a32139b..c21bce3 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -60,6 +60,7 @@ from services import ( nic_status, system, unattend, + upload_verify, wim, ) from services.audit import audit @@ -217,11 +218,32 @@ def images_import(): flash(f"Source path does not exist: {source}", "danger") return redirect(url_for("images_import")) + # Network uploads must be the verified Upload-Image.ps1 upload: every + # file hashed on this disk against the PC's manifest, nothing stray. + up_real = os.path.realpath(config.UPLOAD_DIR) + src_real = os.path.realpath(source) + is_upload = src_real == up_real or src_real.startswith(up_real + "/") + if is_upload: + if src_real != up_real: + flash("Import the whole upload folder; its manifest covers all of it.", "danger") + return redirect(url_for("images_import")) + # Held through gate, move and clean-up: no verify or PUT may + # touch the folder while its files are being moved. + if not upload_verify.try_lock(): + flash("Import refused: a verification or file upload is running. Try again.", "danger") + return redirect(url_for("images_import")) + ok, reason, _info = upload_verify.gate(target, locked=True) + if not ok: + upload_verify.unlock() + flash(f"Import refused: {reason}", "danger") + return redirect(url_for("images_import")) + root = fs.image_root(target) dest = fs.deploy_path(target) try: os.makedirs(dest, exist_ok=True) - src_items = os.listdir(source) + # The manifest / verify state stay behind (finish_import clears them). + src_items = [i for i in os.listdir(source) if not upload_verify.is_reserved(i)] # Move files from network upload to save disk space; copy from USB. # deploy.sync_tree uses rsync --checksum: unchanged files are @@ -265,18 +287,32 @@ def images_import(): media_tag = os.path.join(control_dir, "Media.tag") Path(media_tag).touch() - audit("IMAGE_IMPORT", f"{source} -> {target}") + if is_upload: + rec = upload_verify.finish_import(root, target) + audit("IMAGE_IMPORT", f"{source} -> {target} (verified upload {rec.get('upload_id')})") + else: + audit("IMAGE_IMPORT", f"{source} -> {target} (USB, not verified)") flash( f"Successfully imported content to {config.FRIENDLY_NAMES.get(target, target)}.", "success", ) except Exception as exc: flash(f"Import failed: {exc}", "danger") + finally: + if is_upload: + upload_verify.unlock() return redirect(url_for("images_import")) + up_ok, up_reason, up_info = upload_verify.gate() + last_imports = {it: upload_verify.last_import(fs.image_root(it)) for it in config.IMAGE_TYPES} return render_template( "import.html", + upload_ok=up_ok, + upload_reason=up_reason, + upload_info=up_info, + last_imports=last_imports, + upload_dir=config.UPLOAD_DIR, usb_mounts=usb_mounts, upload_sources=upload_sources, images=image_list, @@ -1872,6 +1908,46 @@ def audit_log(): # Routes - JSON API # --------------------------------------------------------------------------- +@app.route("/images/import/remove-extras", methods=["POST"]) +def images_import_remove_extras(): + try: + n = upload_verify.remove_extras() + except upload_verify.UploadError as ex: + flash(f"Could not remove: {ex}", "danger") + return redirect(url_for("images_import")) + audit("UPLOAD_REMOVE_EXTRAS", f"{n} file(s) not in the manifest removed") + flash(f"Removed {n} file(s) that were not part of this upload.", "success") + return redirect(url_for("images_import")) + + +@app.route("/api/upload/verify", methods=["GET", "POST"]) +def api_upload_verify(): + # CSRF-exempt machine-to-machine endpoint: Upload-Image.ps1 POSTs its + # manifest after robocopy, then polls GET until the hashing is done. + if request.method == "GET": + return jsonify(upload_verify.public_state()) + data = request.get_json(silent=True) + try: + st = upload_verify.start_verify(data) + except upload_verify.UploadError as ex: + return jsonify({"error": str(ex)}), ex.status + audit("UPLOAD_VERIFY", f"upload {st.get('upload_id')}: {st.get('total_files')} file(s)") + return jsonify(st), 202 + + +@app.route("/api/upload/file", methods=["PUT"]) +def api_upload_file(): + # Second channel for files SMB delivered wrong: one manifest file per + # PUT, hash-checked before it replaces anything. + try: + res = upload_verify.receive_file(request.args.get("upload_id", ""), request.args.get("path", ""), + request.stream, request.content_length) + except upload_verify.UploadError as ex: + return jsonify({"error": str(ex)}), ex.status + audit("UPLOAD_FILE_RESENT", f"{res['path']} ({res['size']} bytes) over HTTP") + return jsonify(res), 200 + + @app.route("/api/services") def api_services(): services = {s: system.service_status(s) for s in ("dnsmasq", "apache2", "smbd")} diff --git a/webapp/services/csrf.py b/webapp/services/csrf.py index d04ddc8..accd6ee 100644 --- a/webapp/services/csrf.py +++ b/webapp/services/csrf.py @@ -11,6 +11,7 @@ CSRF_EXEMPT_PATHS = { "/imaging/status", "/reports/cloud-status", "/hardware/seen", + "/api/upload/verify", # Upload-Image.ps1 (PUT /api/upload/file is not a POST) } diff --git a/webapp/services/upload_verify.py b/webapp/services/upload_verify.py new file mode 100644 index 0000000..4347be0 --- /dev/null +++ b/webapp/services/upload_verify.py @@ -0,0 +1,708 @@ +"""Verified MCL uploads: prove every file in the upload folder is the one +the Windows PC sent, repair bad ones over HTTP, and gate the import. + +Upload-Image.ps1 (work PC) hashes every file it copies into +upload-manifest.json and POSTs that manifest here after robocopy. A +background thread then hashes the files on THIS disk (no SMB client cache +in the way) and records which are missing, which differ, and which were +already bad in the MCL cache (their hash on the PC does not match the hash +MCL's own Control JSONs publish). The script re-sends the bad small files +with PUT, which writes them to a temp name, checks the hash and renames +them into place. + +The import page only imports an upload folder whose last verification +passed and whose files have not changed since (size, mtime, ctime and +inode of every file), with no stray files from an earlier upload mixed in. +The import holds the same lock as verification and PUT, so neither can +run while files are being moved. + +Paths are matched against the disk case-insensitively, as Windows and +Samba do: the manifest's case comes from MCL's JSON, the disk's from +whatever robocopy created first. + +State files live at the top of the upload folder and are never imported: + upload-manifest.json the manifest (written by the POST, not by SMB) + .upload-verify.json progress / last result (small; polled) + .upload-verify-files.json per-file hash cache and snapshot (gate only) +""" + +import hashlib +import json +import os +import re +import secrets +import threading +import time +from contextlib import contextmanager +from datetime import datetime, timezone + +import config + +MANIFEST = "upload-manifest.json" +STATE = ".upload-verify.json" +DETAIL = ".upload-verify-files.json" +TOP_FILES = (MANIFEST, STATE, DETAIL) +PART_RE = re.compile(r"^\..+\.part-[0-9a-f]{16}$") + +SMALL_MAX = 64 * 1024 * 1024 # PUT limit; bigger files go back to robocopy +MAX_FILES = 200_000 +UPLOAD_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,80}$") +SHA_RE = re.compile(r"^[0-9a-fA-F]{64}$") +DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]") +CHUNK = 1024 * 1024 + +_lock = threading.Lock() # one verify, PUT or import at a time + + +class UploadError(ValueError): + """Bad request (400) unless .status says otherwise.""" + + def __init__(self, msg, status=400): + super().__init__(msg) + self.status = status + + +@contextmanager +def exclusive(what): + """Hold the upload lock or fail at once with 409.""" + if not _lock.acquire(blocking=False): + raise UploadError(f"busy: a verification, file upload or import is running ({what} refused)", 409) + try: + yield + finally: + _lock.release() + + +# --- paths ------------------------------------------------------------------ + +def root(): + return config.UPLOAD_DIR + + +def _now(): + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +def is_reserved(rel): + """Only our own files at the top of the folder (and their temp files). + A '.part-' file deeper down is debris and counts as stray.""" + if "/" in rel: + return False + if rel in TOP_FILES: + return True + m = PART_RE.match(rel) + return bool(m) and any(rel.startswith(f".{n}.part-") for n in TOP_FILES) + + +def norm_path(p): + """Manifest path -> 'Deploy/Control/x.json'. Raises on anything that + could leave the upload folder.""" + if not isinstance(p, str) or not p or len(p) > 1024 or "\x00" in p: + raise UploadError("bad path") + p = p.replace("\\", "/") + if p.startswith("/") or DRIVE_RE.match(p) or ":" in p: + raise UploadError(f"absolute path not allowed: {p}") + parts = p.split("/") + if any(s in ("", ".", "..") for s in parts): + raise UploadError(f"bad path: {p}") + if is_reserved(p) or PART_RE.match(parts[-1]): + raise UploadError(f"reserved name: {p}") + return p + + +def safe_join(rel, base=None): + """Absolute path of rel inside base, refusing symlink escapes.""" + base = os.path.realpath(base or root()) + full = os.path.join(base, *rel.split("/")) + real = os.path.realpath(full) + if real != full or not real.startswith(base + os.sep): + # realpath differs: a symlink somewhere on the way. Refuse it. + raise UploadError(f"path leaves the upload folder or uses a link: {rel}") + return full + + +def scan(base=None): + """Walk the upload folder once. + Returns (files, links): files = {rel_lower: [rel, ...]} for regular + files; links = rels of symlinks and other non-regular entries, which + are never part of an upload.""" + base = base or root() + files, links = {}, [] + for dp, dns, fns in os.walk(base): + for d in list(dns): + if os.path.islink(os.path.join(dp, d)): + links.append(os.path.relpath(os.path.join(dp, d), base).replace(os.sep, "/")) + dns.remove(d) + dns.sort() + for fn in sorted(fns): + full = os.path.join(dp, fn) + rel = os.path.relpath(full, base).replace(os.sep, "/") + if is_reserved(rel): + continue + if os.path.islink(full) or not os.path.isfile(full): + links.append(rel) + continue + files.setdefault(rel.lower(), []).append(rel) + return files, links + + +def _key(st): + """What 'unchanged' means: robocopy keeps the source mtime, so mtime and + size alone cannot tell a re-copy from the bad copy before it. ctime is + set by the kernel on every write and cannot be set from userland.""" + return [st.st_size, st.st_mtime_ns, st.st_ctime_ns, st.st_ino] + + +# --- manifest --------------------------------------------------------------- + +def parse_manifest(data): + """Validate a manifest dict; return it with normalised paths and + lower-case hashes.""" + if not isinstance(data, dict): + raise UploadError("manifest must be a JSON object") + if data.get("manifest_version") != 1: + raise UploadError("unsupported manifest_version (want 1)") + uid = data.get("upload_id") + if not isinstance(uid, str) or not UPLOAD_ID_RE.match(uid): + raise UploadError("bad upload_id") + files = data.get("files") + if not isinstance(files, list) or not files: + raise UploadError("manifest has no files") + if len(files) > MAX_FILES: + raise UploadError("too many files") + out, seen = [], set() + for f in files: + if not isinstance(f, dict): + raise UploadError("bad file entry") + p = norm_path(f.get("path")) + size, sha = f.get("size"), f.get("sha256") + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + raise UploadError(f"bad size for {p}") + if not isinstance(sha, str) or not SHA_RE.match(sha): + raise UploadError(f"bad sha256 for {p}") + if p.lower() in seen: + raise UploadError(f"duplicate path: {p}") + seen.add(p.lower()) + out.append({"path": p, "size": size, "sha256": sha.lower()}) + mcl = data.get("mcl") if isinstance(data.get("mcl"), dict) else {} + clean = {"manifest_version": 1, "upload_id": uid, "files": out, "mcl": mcl} + for k in ("created", "source_pc"): + if isinstance(data.get(k), str): + clean[k] = data[k][:200] + return clean + + +def _read_json(path): + with open(path, "rb") as f: + return json.loads(f.read().decode("utf-8-sig")) + + +def _write_json(path, obj, indent=1): + """Atomic: temp file in the same folder, fsync, rename.""" + d = os.path.dirname(path) + tmp = os.path.join(d, f".{os.path.basename(path)}.part-{secrets.token_hex(8)}") + with open(tmp, "w", encoding="utf-8") as f: + json.dump(obj, f, indent=indent) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +def load_manifest(): + """(manifest, raw_sha) or (None, None) when there is none.""" + p = os.path.join(root(), MANIFEST) + if not os.path.isfile(p): + return None, None + with open(p, "rb") as f: + raw = f.read() + try: + m = parse_manifest(json.loads(raw.decode("utf-8-sig"))) + except (ValueError, UnicodeDecodeError): + return None, None + return m, hashlib.sha256(raw).hexdigest() + + +def load_state(): + try: + return _read_json(os.path.join(root(), STATE)) + except (OSError, ValueError): + return None + + +def load_detail(): + try: + return _read_json(os.path.join(root(), DETAIL)) + except (OSError, ValueError): + return {} + + +def _chown_like_parent(path): + """Match the share's owner (Samba 'force user = pxe') so a later SMB + re-copy can overwrite what the webapp (root) wrote.""" + try: + st = os.stat(os.path.dirname(path)) + os.chown(path, st.st_uid, st.st_gid) + except (OSError, AttributeError): + pass + + +# --- MCL's own hashes and OS ------------------------------------------------ + +def _mcl_rel(dest_dir, file_name): + """MCL destination ('*destinationdir*\\Deploy\\Packages\\X' or + 'C:\\Deploy\\Languages\\Y') + file name -> 'Deploy/Packages/X/name'.""" + d = (dest_dir or "").replace("\\", "/") + d = re.sub(r"^\*destinationdir\*/?", "", d, flags=re.I) + d = re.sub(r"^[A-Za-z]:/", "", d).strip("/") + return f"{d}/{file_name}" if d else file_name + + +def _control_list(base, name): + try: + d = _read_json(os.path.join(base, "Deploy", "Control", name)) + return d if isinstance(d, list) else [] + except (OSError, ValueError, UnicodeDecodeError): + return [] + + +def mcl_expected(base=None): + """{path_lower: (path, sha256_lower)} for every payload file the + Control JSONs in this upload describe. Only real SHA-256 values.""" + base = base or root() + exp = {} + + def add(dest, name, sha): + if not (dest and name and isinstance(sha, str) and SHA_RE.match(sha)): + return + rel = _mcl_rel(dest, name) + exp[rel.lower()] = (rel, sha.lower()) + + for e in _control_list(base, "OperatingSystem.json"): + wim = ((e or {}).get("operatingSystemVersion") or {}).get("wim") or {} + add(wim.get("destinationDir") or wim.get("DestinationDir"), + wim.get("fileName") or wim.get("FileName") or "install.wim", wim.get("hash") or wim.get("Hash")) + for fname in ("HardwareDriver.json", "packages.json"): + for e in _control_list(base, fname): + if isinstance(e, dict): + add(e.get("destinationDir") or e.get("DestinationDir"), + e.get("fileName") or e.get("FileName"), e.get("hash") or e.get("Hash")) + for e in _control_list(base, "LanguagePacks.json"): + for lf in (e or {}).get("files") or []: + if isinstance(lf, dict): + add(lf.get("destFolder"), lf.get("fileName"), lf.get("hash")) + return exp + + +def detected_os(manifest=None, base=None): + """The OS the upload carries: the OperatingSystem.json entry whose id + is the selected one (manifest mcl.os_id, else Tools/user_selections.json). + A single entry is taken as is. None when it cannot be told.""" + base = base or root() + entries = [e.get("operatingSystemVersion") for e in _control_list(base, "OperatingSystem.json") + if isinstance(e, dict) and isinstance(e.get("operatingSystemVersion"), dict)] + if not entries: + return None + want = ((manifest or {}).get("mcl") or {}).get("os_id") + if want is None: + try: + sel = _read_json(os.path.join(base, "Tools", "user_selections.json")) + want = sel[0].get("OperatingSystemSelection") + except (OSError, ValueError, KeyError, IndexError, TypeError, AttributeError): + want = None + if want is not None: + v = next((e for e in entries if str(e.get("id")) == str(want)), None) + else: + v = entries[0] if len(entries) == 1 else None + if not v or not v.get("productName"): + return None + return {"id": v.get("id"), "name": v.get("productName", ""), + "version": v.get("versionNumber", ""), "build": v.get("buildNumber", "")} + + +def allowed_targets(os_info): + """LTSC is always Shop Floor; anything else is Standard or Engineer. + Unknown OS: nothing (fail closed).""" + if not os_info: + return [] + if "ltsc" in (os_info.get("name") or "").lower(): + return [t for t in config.IMAGE_TYPES if "shopfloor" in t] + return [t for t in config.IMAGE_TYPES if "shopfloor" not in t] + + +# --- verification ----------------------------------------------------------- + +def _sha256(path, progress=None): + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + b = f.read(CHUNK) + if not b: + break + h.update(b) + if progress: + progress(len(b)) + return h.hexdigest() + + +def extras(manifest, base=None, disk=None): + """Files (and links) in the upload folder that this upload did not send.""" + files, links = disk or scan(base) + want = {f["path"].lower() for f in manifest["files"]} + out = list(links) + for low, rels in files.items(): + if low not in want: + out.extend(rels) + elif len(rels) > 1: + out.extend(rels[1:]) # same name in two cases: one is stray + return sorted(out) + + +def run_verify(manifest, manifest_sha, base=None): + """Hash every manifest file on disk; write and return the result. + Call with the lock held.""" + base = base or root() + state_path = os.path.join(base, STATE) + cache = {} + prev = load_detail() if base == root() else {} + if prev.get("upload_id") == manifest["upload_id"]: + cache = prev.get("hashes") or {} + + files = manifest["files"] + total_bytes = sum(f["size"] for f in files) + st = {"state": "running", "upload_id": manifest["upload_id"], "manifest_sha": manifest_sha, + "started": _now(), "total_files": len(files), "total_bytes": total_bytes, + "done_files": 0, "done_bytes": 0} + _write_json(state_path, st) + last = [time.monotonic()] + + def tick(n): + st["done_bytes"] += n + if time.monotonic() - last[0] > 2: + last[0] = time.monotonic() + _write_json(state_path, st) + + disk = scan(base) + missing, mismatch, empty_json, hashes, snap = [], [], [], {}, {} + for f in files: + rel = f["path"] + on_disk = disk[0].get(rel.lower()) or [] + if not on_disk: + missing.append(rel) + st["done_bytes"] += f["size"] + st["done_files"] += 1 + continue + if len(on_disk) > 1: + mismatch.append({"path": rel, "reason": "two files differ only in case: " + ", ".join(on_disk)}) + st["done_bytes"] += f["size"] + st["done_files"] += 1 + continue + actual = on_disk[0] + try: + p = safe_join(actual, base) + except UploadError: + missing.append(rel) + st["done_files"] += 1 + continue + s = os.stat(p) + k = _key(s) + if s.st_size != f["size"]: + mismatch.append({"path": rel, "reason": f"size {s.st_size}, expected {f['size']}"}) + st["done_bytes"] += f["size"] + else: + c = cache.get(actual) + if c and c[:4] == k and c[4] == f["sha256"]: + sha = c[4] # same file, already proven right + st["done_bytes"] += f["size"] + else: + sha = _sha256(p, tick) + if sha != f["sha256"]: + mismatch.append({"path": rel, "reason": "content differs"}) + else: + hashes[actual] = k + [sha] # only proven-good files are cached + snap[actual] = k + low = rel.lower() + if s.st_size == 0 and low.endswith(".json") and low.startswith(("deploy/control/", "tools/")): + empty_json.append(rel) + st["done_files"] += 1 + + # Bad at the source: the PC's hash is not the one MCL publishes. + bad_cache = [] + by_lower = {f["path"].lower(): f for f in files} + for low, (_mrel, msha) in mcl_expected(base).items(): + f = by_lower.get(low) + if f and f["sha256"] != msha: + bad_cache.append({"path": f["path"], "reason": "hash differs from MCL's Control JSON"}) + for rel in empty_json: + bad_cache.append({"path": rel, "reason": "empty Control JSON"}) + + ext = extras(manifest, base, disk) + ok = not (missing or mismatch or bad_cache) + os_info = detected_os(manifest, base) if ok else None + allowed = allowed_targets(os_info) if ok else [] + result = { + "state": "done", "ok": ok, "importable": ok and not ext and bool(allowed), + "upload_id": manifest["upload_id"], "manifest_sha": manifest_sha, + "started": st["started"], "finished": _now(), + "total_files": len(files), "total_bytes": total_bytes, + "missing": missing, + "mismatch": mismatch, + "bad_in_cache": bad_cache, + "extra": ext[:500], "extra_count": len(ext), + "os": os_info, + "allowed_targets": allowed, + "small_max": SMALL_MAX, + } + _write_json(os.path.join(base, DETAIL), {"upload_id": manifest["upload_id"], + "manifest_sha": manifest_sha, + "snapshot": snap, "hashes": hashes}, indent=None) + _write_json(state_path, result) + return result + + +def start_verify(data): + """Store the POSTed manifest and start hashing in the background. + Returns the public state.""" + m = parse_manifest(data) + if not _lock.acquire(blocking=False): + raise UploadError("busy: a verification, file upload or import is running", 409) + try: + base = root() + os.makedirs(base, exist_ok=True) + raw = json.dumps(data, indent=1).encode("utf-8") + mpath = os.path.join(base, MANIFEST) + tmp = os.path.join(base, f".{MANIFEST}.part-{secrets.token_hex(8)}") + with open(tmp, "wb") as f: + f.write(raw) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, mpath) + _chown_like_parent(mpath) + msha = hashlib.sha256(raw).hexdigest() + _write_json(os.path.join(base, STATE), + {"state": "running", "upload_id": m["upload_id"], "manifest_sha": msha, + "started": _now(), "total_files": len(m["files"]), + "total_bytes": sum(f["size"] for f in m["files"]), "done_files": 0, + "done_bytes": 0}) + except BaseException: + _lock.release() + raise + + def work(): + try: + run_verify(m, msha) + except Exception as ex: # report, never leave "running" behind + try: + _write_json(os.path.join(root(), STATE), + {"state": "error", "ok": False, "importable": False, + "upload_id": m["upload_id"], "error": str(ex), "finished": _now()}) + except OSError: + pass + finally: + _lock.release() + + threading.Thread(target=work, name="upload-verify", daemon=True).start() + return public_state() + + +def public_state(): + s = load_state() + return s if s else {"state": "none"} + + +# --- second channel: PUT one small file ------------------------------------- + +def receive_file(upload_id, rel, stream, length): + """Write one manifest file from an HTTP body. Temp name, hash check, + rename. Returns {"path", "sha256", "size"}.""" + rel = norm_path(rel) + with exclusive("file upload"): + m, _ = load_manifest() + if not m: + raise UploadError("no manifest: POST /api/upload/verify first", 409) + if upload_id != m["upload_id"]: + raise UploadError("upload_id does not match the current manifest", 409) + entry = next((f for f in m["files"] if f["path"].lower() == rel.lower()), None) + if not entry: + raise UploadError(f"not in the manifest: {rel}", 404) + if entry["size"] > SMALL_MAX: + raise UploadError(f"{entry['path']} is over {SMALL_MAX // (1024 * 1024)} MB: re-copy it with robocopy", 413) + if length is None or length != entry["size"]: + raise UploadError(f"body is {length} bytes, manifest says {entry['size']}") + # Replace the file that is there, whatever its case; else create it, + # reusing existing folders case-insensitively (as Samba does). + on_disk = scan()[0].get(entry["path"].lower()) or [] + if len(on_disk) > 1: + raise UploadError(f"two files differ only in case: {', '.join(on_disk)}; remove one", 409) + target_rel = on_disk[0] if on_disk else _resolve_dirs(entry["path"]) + dest = safe_join(target_rel) + d = os.path.dirname(dest) + tmp = os.path.join(d, f".{os.path.basename(dest)}.part-{secrets.token_hex(8)}") + h, n = hashlib.sha256(), 0 + try: + with open(tmp, "xb") as f: + while n <= entry["size"]: + b = stream.read(min(CHUNK, entry["size"] + 1 - n)) + if not b: + break + n += len(b) + h.update(b) + f.write(b) + f.flush() + os.fsync(f.fileno()) + if n != entry["size"]: + raise UploadError(f"received {n} bytes, expected {entry['size']}") + if h.hexdigest() != entry["sha256"]: + raise UploadError(f"hash of the received {entry['path']} does not match the manifest", 422) + if os.path.isdir(dest) and not os.path.islink(dest): + raise UploadError(f"a folder is in the way: {target_rel}", 409) + _chown_like_parent(tmp) + os.chmod(tmp, 0o664) + os.replace(tmp, dest) + finally: + if os.path.exists(tmp): + os.remove(tmp) + return {"path": entry["path"], "written_as": target_rel, "sha256": entry["sha256"], "size": n} + + +def _resolve_dirs(rel): + """Map rel's folders onto existing ones case-insensitively, creating + what is missing (refusing links). Returns the path to write.""" + cur = os.path.realpath(root()) + out = [] + for part in rel.split("/")[:-1]: + hit = None + try: + for name in os.listdir(cur): + if name.lower() == part.lower(): + hit = name + break + except OSError: + pass + name = hit or part + nxt = os.path.join(cur, name) + if os.path.islink(nxt): + raise UploadError(f"link in path: {rel}") + if not os.path.isdir(nxt): + os.mkdir(nxt) + _chown_like_parent(nxt) + cur = nxt + out.append(name) + out.append(rel.split("/")[-1]) + return "/".join(out) + + +# --- import gate ------------------------------------------------------------ + +def gate(target=None, locked=False): + """(ok, reason, info) for importing the upload folder into target. + info carries what the import page shows. Pass locked=True when the + caller already holds the lock (the import itself).""" + m, msha = load_manifest() + info = {"manifest": None, "state": public_state()} + if not m: + return False, ("No upload manifest. Upload with scripts\\Upload-Image.ps1, which " + "checks every file; files copied by hand cannot be verified."), info + info["manifest"] = {k: m.get(k) for k in ("upload_id", "created", "source_pc", "mcl")} + info["manifest"]["files"] = len(m["files"]) + info["manifest"]["bytes"] = sum(f["size"] for f in m["files"]) + s = load_state() + if (not locked and _lock.locked()) or (s and s.get("state") == "running"): + return False, "A verification, file upload or import is running.", info + if not s or s.get("state") != "done": + return False, "This upload has not been verified yet (run Upload-Image.ps1 to the end).", info + if s.get("upload_id") != m["upload_id"] or s.get("manifest_sha") != msha: + return False, "The manifest changed after the last verification; verify again.", info + if not s.get("ok"): + n = len(s.get("missing", [])) + len(s.get("mismatch", [])) + len(s.get("bad_in_cache", [])) + return False, f"The last verification found {n} bad file(s).", info + det = load_detail() + if det.get("upload_id") != m["upload_id"] or det.get("manifest_sha") != msha: + return False, "Verification details are missing; verify again.", info + snap = det.get("snapshot") or {} + if len(snap) != len(m["files"]): + return False, "Verification details are incomplete; verify again.", info + for rel, key in snap.items(): + try: + st = os.stat(safe_join(rel)) + except (OSError, UploadError): + return False, f"{rel} is gone since verification; verify again.", info + if _key(st) != key: + return False, f"{rel} changed since verification; verify again.", info + ext = extras(m) + info["extra"] = ext[:50] + info["extra_count"] = len(ext) + if ext: + return False, (f"{len(ext)} file(s) in the upload folder are not part of this upload " + "(left from an earlier one). Remove them first."), info + allowed = s.get("allowed_targets") or [] + info["allowed_targets"] = allowed + info["os"] = s.get("os") + if not allowed: + return False, "Cannot tell which OS this upload carries (OperatingSystem.json), so it has no safe target.", info + if target is not None and target not in allowed: + return False, f"This upload carries {(s.get('os') or {}).get('name', 'an unknown OS')}; it can only go to: {', '.join(allowed)}.", info + return True, "", info + + +def remove_extras(): + """Delete files and links in the upload folder that are not in the + manifest. Returns how many were removed.""" + with exclusive("remove"): + m, _ = load_manifest() + if not m: + raise UploadError("no manifest") + n = 0 + for rel in extras(m): + p = os.path.join(root(), *rel.split("/")) + if os.path.islink(p) or os.path.isfile(p): + os.remove(p) + n += 1 + for dp, _dns, _fns in os.walk(root(), topdown=False): + if dp != root(): + try: + os.rmdir(dp) + except OSError: + pass + return n + + +def import_record(target): + """What gets stored on the image after a verified import.""" + m, _ = load_manifest() + s = load_state() or {} + return {"upload_id": m["upload_id"] if m else None, "imported": _now(), "target": target, + "source_pc": (m or {}).get("source_pc"), "mcl": (m or {}).get("mcl"), + "os": s.get("os"), "files": len(m["files"]) if m else 0} + + +def finish_import(image_root, target): + """After the import moved the files: keep a record on the image, then + clear the manifest and state so the folder starts fresh. Call with + the lock held.""" + rec = import_record(target) + try: + _write_json(os.path.join(image_root, ".mcl-import.json"), rec) + except OSError: + pass + for name in TOP_FILES: + try: + os.remove(os.path.join(root(), name)) + except OSError: + pass + return rec + + +def last_import(image_root): + try: + return _read_json(os.path.join(image_root, ".mcl-import.json")) + except (OSError, ValueError): + return None + + +def try_lock(): + """For the import route: take the lock without waiting (True if held).""" + return _lock.acquire(blocking=False) + + +def unlock(): + _lock.release() diff --git a/webapp/templates/import.html b/webapp/templates/import.html index 0ce8a40..6e843b3 100644 --- a/webapp/templates/import.html +++ b/webapp/templates/import.html @@ -21,6 +21,12 @@ .imp-actions { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; } .imp-empty { display: flex; flex-direction: column; gap: 10px; align-items: flex-start; font-size: 14px; color: var(--text-2); line-height: 1.6; } .imp-empty p { margin: 0; } + .upv-rows { display: flex; flex-direction: column; gap: 6px; font-size: 14px; } + .upv-row { display: grid; grid-template-columns: 92px minmax(0, 1fr); gap: 10px; align-items: center; } + .upv-row .k { color: var(--muted); font-size: 13px; } + .upv-bad summary { cursor: pointer; font-size: 13px; font-weight: 600; } + .upv-bad ul { margin: 8px 0 0; padding-left: 18px; display: flex; flex-direction: column; gap: 3px; font-size: 13px; } + .upv-bad li { min-width: 0; } .ig { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.3fr) 92px 112px auto; align-items: center; gap: 0 14px; padding: 11px 24px; font-size: 14px; border-top: 1px solid var(--line); } @@ -50,40 +56,81 @@

From the network upload share

-

Files copied over SMB to \\172.16.9.1\image-upload. They are moved, not copied.

+

Uploaded by Upload-Image.ps1 to \\172.16.9.1\image-upload and checked file by file. They are moved, not copied.

- {% if upload_sources %} + {% set man = upload_info.manifest %} + {% set st = upload_info.state %} + {% if man %} +
+
Upload{{ man.upload_id }}
+ {% if man.mcl %} +
MCL cache{{ man.mcl.os_name or 'Unknown OS' }}{% if man.mcl.os_version %} {{ man.mcl.os_version }}{% endif %}, finished {{ (man.mcl.finished or '?')|replace('T', ' ') }}
+ {% endif %} +
Files{{ man.files }} ({{ man.bytes|filesizeformat(true) }})
+
Verified + {% if st.state == 'running' %}{{ ui.chip('Checking ' ~ st.done_files ~ '/' ~ st.total_files, 'warn') }} + {% elif st.state == 'done' and st.ok %}{{ ui.chip('Every file matches', 'ok') }} {{ (st.finished or '')|replace('T', ' ') }} + {% elif st.state == 'done' %}{{ ui.chip((st.missing|length + st.mismatch|length + st.bad_in_cache|length) ~ ' bad file(s)', 'err') }} + {% elif st.state == 'error' %}{{ ui.chip('Check failed', 'err') }} {{ st.error }} + {% else %}{{ ui.chip('Not verified', 'off') }}{% endif %} +
+
+ {% if st.state == 'done' and not st.ok %} +
Bad files +
    + {% for f in st.missing %}
  • Missing: {{ f }}
    • {% endfor %} + {% for f in st.mismatch %}
  • Differs: {{ f.path }}
    • {% endfor %} + {% for f in st.bad_in_cache %}
  • Bad in MCL cache: {{ f.path }}
    • {% endfor %} +
+

Run Upload-Image.ps1 again: it re-sends the files that differ. Files bad in the MCL cache must be downloaded again by MCL.

+
+ {% endif %} + {% endif %} + + {% if upload_info.extra_count %} +
{{ upload_info.extra_count }} file(s) in the upload folder are not part of this upload, left from an earlier one. They would be imported with it.
+
Show them
    + {% for f in upload_info.extra %}
  • {{ f }}
    • {% endfor %} + {% if upload_info.extra_count > upload_info.extra|length %}
  • and {{ upload_info.extra_count - upload_info.extra|length }} more
    • {% endif %} +
+
+ + +
+ {% endif %} + + {% if upload_ok %}
-
- - -
+
+ {% set allowed = upload_info.allowed_targets %} - Content goes into the image's Deploy folder. Out-of-box Drivers are pooled once in a shared folder and linked from each image to save disk space. + {% if upload_info.os %}This upload carries {{ upload_info.os.name }}{% if upload_info.os.version %} {{ upload_info.os.version }}{% endif %}, so it can only go to the image(s) listed.{% endif %} + Out-of-box Drivers are pooled once in a shared folder and linked from each image.
+
The files are moved out of the upload folder, so the source is used up. Files with the same name in the image are overwritten. Large images take several minutes.
+ {% elif upload_sources %} +
Import blocked: {{ upload_reason }}
+
{% else %}
{{ ui.chip('Nothing uploaded', 'off') }} -

Map \\172.16.9.1\image-upload on your Windows PC and copy the Deploy folder contents there, then refresh.

+

On the PC with the Media Creator Lite cache, run scripts\Upload-Image.ps1. It copies the cache here and checks every file.

{% endif %} @@ -152,6 +199,15 @@ {% block extra_scripts %}