From 51edf98e7d0a269296ff94aadabf5b14759d813f Mon Sep 17 00:00:00 2001 From: cproudlock Date: Sun, 14 Jun 2026 11:20:32 -0400 Subject: [PATCH] heal: verify file completeness (size/timestamp), not just presence The shallow present-check passed a file that merely existed, so a partially transferred payload (e.g. a truncated PC-DMIS MSI) looked PRESENT and was never re-pulled - then failed to install because it was incomplete. Replace it with a per-item robocopy that compares size + timestamp on every file and re-pulls anything missing OR partial, skipping ones already complete. VerifyOnly uses /L to report INCOMPLETE without changing anything. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Verify-And-Heal-Staging.ps1 | 93 +++++++++---------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/playbook/shopfloor-setup/Verify-And-Heal-Staging.ps1 b/playbook/shopfloor-setup/Verify-And-Heal-Staging.ps1 index f9e2bd0..69961e9 100644 --- a/playbook/shopfloor-setup/Verify-And-Heal-Staging.ps1 +++ b/playbook/shopfloor-setup/Verify-And-Heal-Staging.ps1 @@ -17,8 +17,10 @@ Designed to be: - called from the pre-install phase before 00-PreInstall-MachineApps so a bay is never left under-provisioned. -Idempotent. Verifies first; only heals what is actually missing/empty. Heals use -/R:3 /W:5 (resilient), not the WinPE fail-fast /R:1 /W:1. +Idempotent. Uses robocopy per item, which compares size + timestamp on every +file, so it re-pulls anything MISSING or PARTIAL (e.g. a truncated MSI that +"exists" but is incomplete and would fail to install) and skips files already +complete. Heals use /R:3 /W:5 (resilient), not the WinPE fail-fast /R:1 /W:1. Share + creds: read from C:\Enrollment\fetch-source.txt (line1=UNC, line2=user, line3=pass) - same file Fetch-StagingPayload uses - else the defaults below. @@ -88,68 +90,61 @@ if ($pcType -eq 'gea-shopfloor-cmm') { } } -# --- present check ------------------------------------------------------------- -function Test-Present($it){ - if (-not (Test-Path -LiteralPath $it.Verify)) { return $false } - if ($it.Mode -eq 'Dir') { - $n = @(Get-ChildItem -LiteralPath $it.Verify -Recurse -File -EA SilentlyContinue).Count - return ($n -gt 0) - } - return $true -} - -# --- mount share fresh (only if we will heal) --------------------------------- +# --- robocopy-based verify/heal ----------------------------------------------- +# Presence alone is NOT trusted: a partially transferred file (e.g. a truncated +# MSI) exists but is incomplete and breaks install. Instead robocopy runs per +# item and compares size + timestamp on EVERY file, re-pulling any that are +# missing OR differ (partial/truncated) and skipping ones already complete (a +# cheap metadata scan). So it scans all files, not just checks a folder is +# non-empty. VerifyOnly adds /L (list-only): it reports what WOULD be re-pulled +# without changing anything. $drive='Z:'; $mounted=$false function Mount-Share { & net use $drive /delete /y 2>$null | Out-Null; & net use $drive $ShareUnc /user:$ShareUser $SharePass /persistent:no 2>&1 | Out-Null; return ($LASTEXITCODE -eq 0) } -# --- first pass: verify -------------------------------------------------------- $report = New-Object System.Collections.Generic.List[object] -$missing = New-Object System.Collections.Generic.List[object] -foreach ($it in $items) { - if (Test-Present $it) { $report.Add([pscustomobject]@{Item=$it.Label;Status='PRESENT'}) } - else { $missing.Add($it); $report.Add([pscustomobject]@{Item=$it.Label;Status=$(if($it.Optional){'MISSING(opt)'}else{'MISSING'})}) } -} - -# --- heal missing -------------------------------------------------------------- -if ($missing.Count -gt 0 -and -not $VerifyOnly) { - Log "$($missing.Count) item(s) missing - mounting share to heal" - for ($a=1;$a -le 5 -and -not $mounted;$a++){ if (Mount-Share){$mounted=$true;Log "Mounted $ShareUnc as $drive"} else {Log "mount attempt $a failed - 10s" 'WARN'; Start-Sleep 10} } - if (-not $mounted) { Log "Could not mount share - cannot heal. $($missing.Count) item(s) remain missing." 'ERROR' } - else { - foreach ($it in $missing) { - $src = Join-Path $drive $it.Src - if (-not (Test-Path -LiteralPath $src)) { - Log "[heal SKIP] $($it.Label): not on share ($src)$(if($it.Optional){' - optional'})" $(if($it.Optional){'INFO'}else{'WARN'}) - ($report | Where-Object { $_.Item -eq $it.Label })[0].Status = $(if($it.Optional){'ABSENT(opt)'}else{'NO-SOURCE'}) - continue - } - if (-not (Test-Path -LiteralPath $it.Dst)) { New-Item -ItemType Directory -Path $it.Dst -Force | Out-Null } - $args=@($src,$it.Dst) - if ($it.Mode -eq 'Dir') { $args+='/E' } else { $args+=$it.Files } - if ($it.Xd) { $args+=@('/XD',(Join-Path $src $it.Xd)) } - $args+=@('/R:3','/W:5','/NFL','/NDL','/NP') - Log "[heal] $($it.Label): robocopy $src -> $($it.Dst)" - & robocopy @args | Out-Null - $rc=$LASTEXITCODE - Start-Sleep 1 - $ok = Test-Present $it - ($report | Where-Object { $_.Item -eq $it.Label })[0].Status = $(if($ok){'HEALED'}elseif($rc -ge 8){'HEAL-FAIL'}else{'STILL-MISSING'}) - Log "[heal $(if($ok){'OK'}else{'FAIL'})] $($it.Label) robocopy exit=$rc present=$ok" +for ($a=1; $a -le 5 -and -not $mounted; $a++){ if (Mount-Share){$mounted=$true;Log "Mounted $ShareUnc as $drive"} else {Log "mount attempt $a/5 failed - 10s" 'WARN'; Start-Sleep 10} } +if (-not $mounted) { + Log "Could not mount $ShareUnc after 5 attempts - cannot verify/heal. Bay may be under-provisioned; re-run once the share is reachable." 'ERROR' + foreach ($it in $items) { $report.Add([pscustomobject]@{Item=$it.Label;Status='NO-MOUNT'}) } +} else { + foreach ($it in $items) { + $src = Join-Path $drive $it.Src + if (-not (Test-Path -LiteralPath $src)) { + $report.Add([pscustomobject]@{Item=$it.Label;Status=$(if($it.Optional){'ABSENT(opt)'}else{'NO-SOURCE'})}) + Log "[$($it.Label)] source not on share ($src)$(if($it.Optional){' - optional'})" $(if($it.Optional){'INFO'}else{'WARN'}) + continue } - & net use $drive /delete /y 2>$null | Out-Null + if (-not (Test-Path -LiteralPath $it.Dst)) { New-Item -ItemType Directory -Path $it.Dst -Force | Out-Null } + $args=@($src,$it.Dst) + if ($it.Mode -eq 'Dir') { $args+='/E' } else { $args+=$it.Files } + if ($it.Xd) { $args+=@('/XD',(Join-Path $src $it.Xd)) } + $args+=@('/R:3','/W:5','/NFL','/NDL','/NP') + if ($VerifyOnly) { $args+='/L' } # list-only: detect missing/partial, change nothing + $out = & robocopy @args 2>&1 + $rc = $LASTEXITCODE + # robocopy exit bits: 1=copied, 2=extra, 4=mismatch, 8+=failure (<8 success). + $copied = (($rc -band 1) -ne 0) -or (($rc -band 4) -ne 0) + $files = ($out | Select-String -Pattern '^\s*Files :' | Select-Object -First 1) + if ($rc -ge 8) { $status='HEAL-FAIL' } + elseif (-not $copied) { $status='COMPLETE' } # in sync, nothing to do + elseif ($VerifyOnly) { $status='INCOMPLETE' } # would re-pull (missing/partial) + else { $status='HEALED' } # actually re-pulled missing/partial + $report.Add([pscustomobject]@{Item=$it.Label;Status=$status}) + Log "[$($it.Label)] robocopy rc=$rc -> $status $(("$files").Trim())" } + & net use $drive /delete /y 2>$null | Out-Null } # --- report -------------------------------------------------------------------- Log '================ STAGING VERIFY/HEAL REPORT ================' foreach ($r in $report) { Log (" {0,-26} {1}" -f $r.Item, $r.Status) } -$bad = @($report | Where-Object { $_.Status -in @('MISSING','NO-SOURCE','HEAL-FAIL','STILL-MISSING') }) +$bad = @($report | Where-Object { $_.Status -in @('NO-SOURCE','HEAL-FAIL','NO-MOUNT','INCOMPLETE') }) if ($bad.Count -gt 0) { - Log "RESULT: $($bad.Count) REQUIRED item(s) still missing: $(($bad|ForEach-Object{$_.Item}) -join ', ')" 'ERROR' + Log "RESULT: $($bad.Count) item(s) need attention: $(($bad|ForEach-Object{$_.Item+'='+$_.Status}) -join ', ')" 'ERROR' Log "Log: $log" exit 1 } else { - Log 'RESULT: all required payloads present (or healed).' + Log 'RESULT: all required payloads complete (or healed).' Log "Log: $log" exit 0 }