pxe-server/webapp/static/Sync-BPRT.ps1

<#
.SYNOPSIS
    Upload the new and changed files of the vendor BPRT folder to the PXE server.

.DESCRIPTION
    Run on the GE work PC where MediaCreator Lite keeps the vendor's BPRT
    folder (the MCL provisioning packages, the Office zips and the bulk-enroll
    scripts). The PXE server is reachable from there only over the GE network:
    its webapp and SSH, no SMB.

      1. GET <WebUrl>/updates/bprt/manifest.json: path, size and sha256 of every
         vendor file in the server's live BPRT. The first request after a big
         change makes the server hash 7.7 GB; it answers 503 until then and
         this script waits.
      2. Hash the local folder (Get-FileHash SHA256) and parse-check every
         PowerShell file (a corrupt MediaCreator cache once held scripts with
         stale tails), and check every .ppkg has a WIM header.
      3. Write sync.json (the complete intended vendor file list with sizes and
         hashes) and upload the new and changed files plus sync.json with ONE
         sftp session into BPRT-incoming/<batch>.part on the server, renamed to
         <batch> when complete. You type the password once.
      4. Print the webapp page where the batch is validated and activated.

    Nothing on the server changes until the batch is activated on that page.
    The site's own files in BPRT (our Select-PCConfig.ps1 shim, the
    Register-BulkEnrollTask*.ps1 scripts) are not in the manifest and are
    never uploaded; the vendor's Select-PCConfig.ps1 is compared with the copy
    the server keeps as Select-PCConfig-vendor.ps1.

    Windows PowerShell 5.1, built-in modules and the built-in OpenSSH client
    (sftp.exe) only.

.PARAMETER Source
    The vendor BPRT folder. Default: the MediaCreator Lite cache.

.PARAMETER WebUrl
    The PXE webapp on the GE network.

.PARAMETER SshHost
    The PXE server's GE-network address for sftp.

.PARAMETER SshUser
    SSH login (password prompt).

.PARAMETER DryRun
    Compare and write sync.json and the sftp batch file, print what would be
    uploaded, and stop before connecting with sftp. The manifest is still
    read from the webapp when it answers (read-only); use -ManifestFile to
    work fully offline.

.PARAMETER ZipOut
    Instead of sftp, write the changed files plus sync.json into this zip, to
    upload on the webapp page (the fallback when SSH is not available).

.PARAMETER ManifestFile
    Use a saved manifest.json instead of fetching it.

.PARAMETER Force
    Upload even when a script fails to parse or a package has no WIM header.
    The webapp still refuses to activate such a batch.

.PARAMETER MaxWaitMinutes
    How long to wait for the server to finish hashing its BPRT.

.PARAMETER UseSystemProxy
    Reach the webapp through the system proxy (default: direct, as for any
    10.x address).

.EXAMPLE
    powershell -ExecutionPolicy Bypass -File .\Sync-BPRT.ps1 -DryRun

.EXAMPLE
    powershell -ExecutionPolicy Bypass -File .\Sync-BPRT.ps1

.EXAMPLE
    powershell -ExecutionPolicy Bypass -File .\Sync-BPRT.ps1 -ZipOut C:\Temp\bprt.zip
#>

[CmdletBinding()]
param(
    [string]$Source = 'C:\ProgramData\GEAerospace\MediaCenter\Cache\Deploy\Applications\BPRT',
    # Filled in with the address the page was opened from when the script is
    # downloaded from the PXE webapp (Updates > BPRT sync); else pass them.
    [string]$WebUrl = '',
    [string]$SshHost = '',
    [string]$SshUser = 'pxe',
    [switch]$DryRun,
    [string]$ZipOut,
    [string]$ManifestFile,
    [switch]$Force,
    [int]$MaxWaitMinutes = 30,
    [switch]$UseSystemProxy
)

$ErrorActionPreference = 'Stop'
if (-not $WebUrl -or -not $SshHost) {
    throw 'Give -WebUrl http://<PXE server>:9009 and -SshHost <PXE server> (or download this script from the PXE webapp, which fills them in).'
}

# The cache folder has also been seen under MediaCreator rather than
# MediaCenter; try that one when the default is missing.
$AltSource = 'C:\ProgramData\GEAerospace\MediaCreator\Cache\Deploy\Applications\BPRT'
# Files never sent: Windows litter. Names with [ or ] would be read as
# wildcards by sftp's put.
$SkipNames = @('Thumbs.db', 'desktop.ini', '.DS_Store')
$ScriptExts = @('.ps1', '.psm1', '.psd1')
$WimMagic = [byte[]](0x4D, 0x53, 0x57, 0x49, 0x4D, 0x00, 0x00, 0x00)   # "MSWIM\0\0\0"

function Write-Step([string]$Text) { Write-Host ''; Write-Host "== $Text" -ForegroundColor Cyan }

function Format-Size([double]$Bytes) {
    if ($Bytes -ge 1GB) { return ('{0:N2} GB' -f ($Bytes / 1GB)) }
    if ($Bytes -ge 1MB) { return ('{0:N0} MB' -f ($Bytes / 1MB)) }
    if ($Bytes -ge 1KB) { return ('{0:N0} KB' -f ($Bytes / 1KB)) }
    return ('{0} B' -f $Bytes)
}

function Get-Prefix([string]$Name) {
    # Tenant_Environment_Purpose, the part the enroll scripts match on.
    return (($Name -split '_')[0..2] -join '_')
}

function Get-PpkgVersion([string]$Name) {
    if ($Name -match '_(v\d+\.\d+)\.ppkg$') { return $Matches[1] }
    return '?'
}

function Find-WebException($Err) {
    $ex = $Err.Exception
    while ($ex -and -not ($ex -is [System.Net.WebException])) { $ex = $ex.InnerException }
    return $ex
}

function Get-Manifest {
    param([string]$Url)
    $deadline = (Get-Date).AddMinutes($MaxWaitMinutes)
    while ($true) {
        $req = [System.Net.HttpWebRequest]::Create($Url)
        $req.Timeout = 60000
        $req.ReadWriteTimeout = 60000
        if (-not $UseSystemProxy) { $req.Proxy = $null }
        $resp = $null
        try {
            $resp = $req.GetResponse()
        } catch {
            $wex = Find-WebException $_
            if ($wex -and $wex.Response) { $resp = $wex.Response }
            else { throw "Cannot reach $Url : $($_.Exception.Message)" }
        }
        $status = [int]$resp.StatusCode
        $retry = 30
        $ra = $resp.Headers['Retry-After']
        if ($ra -and ($ra -as [int])) { $retry = [int]$ra }
        try {
            $reader = New-Object System.IO.StreamReader($resp.GetResponseStream(), [System.Text.Encoding]::UTF8)
            $body = $reader.ReadToEnd()
            $reader.Close()
        } finally {
            $resp.Close()
        }
        if ($status -eq 200) { return ($body | ConvertFrom-Json) }
        if ($status -eq 503) {
            $msg = 'The server is busy.'
            try {
                $j = $body | ConvertFrom-Json
                if ($j.message) { $msg = $j.message }
                if ($null -ne $j.percent) { $msg = "$msg ($($j.percent)%)" }
            } catch { }
            if ((Get-Date) -gt $deadline) {
                throw "Gave up after $MaxWaitMinutes minutes: $msg Run this again later."
            }
            Write-Host ("  {0} Asking again in {1} s." -f $msg, $retry)
            Start-Sleep -Seconds $retry
            continue
        }
        throw "HTTP $status from $Url : $body"
    }
}

function ConvertTo-SftpPath([string]$Path) {
    # sftp batch arguments: forward slashes (the Windows client takes them),
    # in double quotes so spaces survive ("M365 Office x86.zip").
    return '"' + ($Path -replace '\\', '/') + '"'
}

function Write-Utf8NoBom([string]$Path, [string]$Text) {
    [System.IO.File]::WriteAllText($Path, $Text, (New-Object System.Text.UTF8Encoding($false)))
}

try {
    # --- source folder ----------------------------------------------------------
    if (-not (Test-Path -LiteralPath $Source -PathType Container)) {
        if (-not $PSBoundParameters.ContainsKey('Source') -and (Test-Path -LiteralPath $AltSource -PathType Container)) {
            $Source = $AltSource
        } else {
            throw "Source folder not found: $Source (use -Source)"
        }
    }
    $Source = (Resolve-Path -LiteralPath $Source).ProviderPath.TrimEnd('\', '/')
    $WebUrl = $WebUrl.TrimEnd('/')
    Write-Host "Source:  $Source"
    Write-Host "Server:  $WebUrl (sftp $SshUser@$SshHost)"

    # --- server manifest --------------------------------------------------------
    Write-Step 'Reading the server manifest'
    $manifest = $null
    if ($ManifestFile) {
        $manifest = Get-Content -LiteralPath $ManifestFile -Raw -Encoding UTF8 | ConvertFrom-Json
        Write-Host "  from $ManifestFile"
    } else {
        try {
            $manifest = Get-Manifest -Url "$WebUrl/updates/bprt/manifest.json"
        } catch {
            if (-not $DryRun) { throw }
            Write-Warning "$($_.Exception.Message)"
            Write-Warning 'Dry run without a manifest: every file counts as new.'
        }
    }
    $server = @{}          # vendor path -> manifest row (PowerShell hashtables ignore case, like Windows)
    $neverSend = @{}       # site files on the server, never uploaded
    $incoming = '/srv/samba/winpeapps/_shared/Applications/BPRT-incoming'
    $finishUrl = "$WebUrl/updates/bprt/"
    if ($manifest) {
        foreach ($f in @($manifest.files)) { if ($f) { $server[$f.path] = $f } }
        foreach ($n in @($manifest.overlay)) { if ($n) { $neverSend[$n] = $true } }
        if ($manifest.incoming) { $incoming = $manifest.incoming }
        if ($manifest.finish_url) { $finishUrl = $manifest.finish_url }
        Write-Host ("  {0} vendor files on the server, {1}, manifest of {2}" -f $server.Count, (Format-Size $manifest.bytes), $manifest.generated)
    }

    # --- local files ------------------------------------------------------------
    Write-Step 'Hashing the local folder'
    $all = @(Get-ChildItem -LiteralPath $Source -Recurse -File -Force | Sort-Object FullName)
    $local = New-Object System.Collections.Generic.List[object]
    $skipped = New-Object System.Collections.Generic.List[string]
    $badNames = New-Object System.Collections.Generic.List[string]
    $totalBytes = [double]0
    foreach ($fi in $all) {
        $rel = $fi.FullName.Substring($Source.Length).TrimStart('\', '/') -replace '\\', '/'
        if ($SkipNames -contains $fi.Name) { $skipped.Add($rel); continue }
        if ($neverSend.ContainsKey($rel)) { $skipped.Add("$rel (site file on the server)"); continue }
        if ($rel -match '[\[\]]') { $badNames.Add($rel) }
        $local.Add([pscustomobject]@{ Path = $rel; Full = $fi.FullName; Size = [int64]$fi.Length; Sha = '' })
        $totalBytes += $fi.Length
    }
    if ($local.Count -eq 0) { throw "No files in $Source" }
    $done = [double]0
    $sw = [System.Diagnostics.Stopwatch]::StartNew()
    foreach ($f in $local) {
        $pct = 0
        if ($totalBytes -gt 0) { $pct = [int](100 * $done / $totalBytes) }
        Write-Progress -Activity 'Hashing BPRT' -Status ("{0} ({1})" -f $f.Path, (Format-Size $f.Size)) -PercentComplete $pct
        $f.Sha = (Get-FileHash -LiteralPath $f.Full -Algorithm SHA256).Hash.ToUpperInvariant()
        $done += $f.Size
    }
    Write-Progress -Activity 'Hashing BPRT' -Completed
    Write-Host ("  {0} files, {1}, hashed in {2:N0} s" -f $local.Count, (Format-Size $totalBytes), $sw.Elapsed.TotalSeconds)
    foreach ($s in $skipped) { Write-Host "  skipped $s" }

    # --- sanity checks ----------------------------------------------------------
    Write-Step 'Checking scripts and packages'
    $problems = New-Object System.Collections.Generic.List[object]
    foreach ($f in $local) {
        $ext = [System.IO.Path]::GetExtension($f.Path).ToLowerInvariant()
        if ($ScriptExts -contains $ext) {
            $tokens = $null
            $errs = $null
            [void][System.Management.Automation.Language.Parser]::ParseFile($f.Full, [ref]$tokens, [ref]$errs)
            if ($errs -and $errs.Count -gt 0) {
                $first = $errs[0]
                $problems.Add([ordered]@{ path = $f.Path; error = ("line {0}: {1} ({2} error(s))" -f $first.Extent.StartLineNumber, $first.Message, $errs.Count) })
            }
        } elseif ($ext -eq '.ppkg') {
            $head = New-Object byte[] 8
            $fs = [System.IO.File]::OpenRead($f.Full)
            try { $n = $fs.Read($head, 0, 8) } finally { $fs.Close() }
            $ok = ($n -eq 8)
            for ($i = 0; $ok -and $i -lt 8; $i++) { if ($head[$i] -ne $WimMagic[$i]) { $ok = $false } }
            if (-not $ok) { $problems.Add([ordered]@{ path = $f.Path; error = 'no WIM header: not a provisioning package' }) }
        }
    }
    foreach ($p in $problems) { Write-Host ("  PROBLEM {0}: {1}" -f $p.path, $p.error) -ForegroundColor Red }
    foreach ($b in $badNames) { Write-Host "  PROBLEM $b : sftp would read [ ] in the name as a wildcard" -ForegroundColor Red }
    if ($problems.Count -eq 0 -and $badNames.Count -eq 0) { Write-Host '  all scripts parse, all packages have a WIM header' }
    if ($badNames.Count -gt 0) { throw 'Rename the files listed above, or upload a zip on the webapp page.' }
    if ($problems.Count -gt 0 -and -not $Force) {
        throw ('{0} file(s) look corrupt. If the MediaCreator cache is damaged, delete it and let MediaCreator Lite download again. -Force uploads anyway, but the server refuses such a batch; it is only for looking at the files there.' -f $problems.Count)
    }

    # --- compare ----------------------------------------------------------------
    $changed = New-Object System.Collections.Generic.List[object]
    $localPaths = @{}
    foreach ($f in $local) {
        $localPaths[$f.Path] = $true
        $s = $server[$f.Path]
        $same = $false
        if ($s) {
            $same = ($f.Sha -eq ([string]$s.sha256).ToUpperInvariant())
            if (-not $same -and $s.sha256_nobom) { $same = ($f.Sha -eq ([string]$s.sha256_nobom).ToUpperInvariant()) }
        }
        if (-not $same) { $changed.Add($f) }
    }
    $removed = @($server.Keys | Where-Object { -not $localPaths.ContainsKey($_) } | Sort-Object)
    $uploadBytes = [double]0
    foreach ($f in $changed) { $uploadBytes += $f.Size }

    Write-Step 'Differences'
    foreach ($f in $changed) {
        $tag = 'changed'
        if (-not $server.ContainsKey($f.Path)) { $tag = 'new' }
        Write-Host ("  {0,-8} {1}  ({2})" -f $tag, $f.Path, (Format-Size $f.Size))
    }
    foreach ($r in $removed) { Write-Host ("  {0,-8} {1}  (not in the local folder; activation removes it from BPRT)" -f 'removed', $r) -ForegroundColor Yellow }
    # Package versions per tenant/environment/purpose.
    $oldPk = @{}
    $newPk = @{}
    foreach ($k in $server.Keys) { if ($k -like '*.ppkg') { $oldPk[(Get-Prefix $k)] = @($oldPk[(Get-Prefix $k)]) + $k } }
    foreach ($f in $local) { if ($f.Path -like '*.ppkg') { $newPk[(Get-Prefix $f.Path)] = @($newPk[(Get-Prefix $f.Path)]) + $f.Path } }
    $prefixes = @(@($oldPk.Keys) + @($newPk.Keys) | Sort-Object -Unique)
    foreach ($p in $prefixes) {
        $o = (@($oldPk[$p] | Where-Object { $_ }) | ForEach-Object { Get-PpkgVersion $_ }) -join ', '
        $n = (@($newPk[$p] | Where-Object { $_ }) | ForEach-Object { Get-PpkgVersion $_ }) -join ', '
        if ($o -ne $n) {
            if (-not $o) { $o = 'none' }
            if (-not $n) { $n = 'none' }
            Write-Host ("  package  {0}: {1} to {2}" -f $p, $o, $n) -ForegroundColor Green
        }
    }
    Write-Host ("  {0} new or changed ({1}), {2} removed, {3} unchanged" -f $changed.Count, (Format-Size $uploadBytes), $removed.Count, ($local.Count - $changed.Count))

    if ($manifest -and $changed.Count -eq 0 -and $removed.Count -eq 0) {
        Write-Host ''
        Write-Host 'The server already has exactly this BPRT. Nothing to upload.' -ForegroundColor Green
        exit 0
    }

    # --- sync.json and the sftp batch ------------------------------------------
    $computer = ($env:COMPUTERNAME -replace '[^A-Za-z0-9-]', '')
    if (-not $computer) { $computer = 'pc' }
    $batch = '{0}-{1}' -f (Get-Date -Format 'yyyyMMdd-HHmmss'), $computer
    $tempRoot = $env:TEMP
    if (-not $tempRoot) { $tempRoot = [System.IO.Path]::GetTempPath() }
    $work = Join-Path (Join-Path $tempRoot 'Sync-BPRT') $batch
    New-Item -ItemType Directory -Path $work -Force | Out-Null

    $fileRows = @($local | ForEach-Object { [ordered]@{ path = $_.Path; size = $_.Size; sha256 = $_.Sha } })
    $sync = [ordered]@{
        format             = 1
        tool               = 'Sync-BPRT.ps1'
        source             = $Source
        computer           = $env:COMPUTERNAME
        user               = $env:USERNAME
        created            = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
        manifest_generated = $(if ($manifest) { $manifest.generated } else { '' })
        files              = $fileRows
        uploaded           = @($changed | ForEach-Object { $_.Path })
        removed            = $removed
        parse_errors       = $problems.ToArray()   # @() of an empty List fails: "Argument types do not match"
    }
    $syncPath = Join-Path $work 'sync.json'
    Write-Utf8NoBom $syncPath ($sync | ConvertTo-Json -Depth 6)

    if ($ZipOut) {
        Write-Step "Writing $ZipOut"
        Add-Type -AssemblyName System.IO.Compression
        Add-Type -AssemblyName System.IO.Compression.FileSystem
        $zipFull = [System.IO.Path]::GetFullPath($ZipOut)
        if (Test-Path -LiteralPath $zipFull) { Remove-Item -LiteralPath $zipFull -Force }
        $zip = [System.IO.Compression.ZipFile]::Open($zipFull, [System.IO.Compression.ZipArchiveMode]::Create)
        try {
            foreach ($f in $changed) {
                Write-Host "  + $($f.Path)"
                # Packages and zips do not compress; storing them saves minutes.
                $level = [System.IO.Compression.CompressionLevel]::Optimal
                if ($f.Path -match '\.(ppkg|zip|exe|png)$') { $level = [System.IO.Compression.CompressionLevel]::NoCompression }
                [void][System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $f.Full, $f.Path, $level)
            }
            [void][System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $syncPath, 'sync.json', [System.IO.Compression.CompressionLevel]::Optimal)
        } finally {
            $zip.Dispose()
        }
        Write-Host ''
        Write-Host ("Wrote {0} ({1})." -f $zipFull, (Format-Size (Get-Item -LiteralPath $zipFull).Length)) -ForegroundColor Green
        Write-Host "Upload it under 'Upload a zip instead' on $finishUrl"
        exit 0
    }

    $remote = "$($incoming.TrimEnd('/'))/$batch"
    $part = "$remote.part"
    $lines = New-Object System.Collections.Generic.List[string]
    $lines.Add('progress')                                   # batch mode starts with the meter off
    $lines.Add('-mkdir ' + (ConvertTo-SftpPath $incoming))   # leading '-': an existing folder is fine
    $lines.Add('mkdir ' + (ConvertTo-SftpPath $part))
    $dirs = @($changed | ForEach-Object { $d = [System.IO.Path]::GetDirectoryName($_.Path); if ($d) { $d -replace '\\', '/' } } | Sort-Object -Unique)
    $made = @{}
    foreach ($d in $dirs) {
        $acc = ''
        foreach ($piece in ($d -split '/')) {
            if ($acc) { $acc = "$acc/$piece" } else { $acc = $piece }
            if (-not $made.ContainsKey($acc)) { $made[$acc] = $true; $lines.Add('-mkdir ' + (ConvertTo-SftpPath "$part/$acc")) }
        }
    }
    foreach ($f in $changed) {
        $lines.Add('put ' + (ConvertTo-SftpPath $f.Full) + ' ' + (ConvertTo-SftpPath "$part/$($f.Path)"))
    }
    # sync.json last, then the rename: the server only treats a folder without
    # .part as a finished upload.
    $lines.Add('put ' + (ConvertTo-SftpPath $syncPath) + ' ' + (ConvertTo-SftpPath "$part/sync.json"))
    $lines.Add('rename ' + (ConvertTo-SftpPath $part) + ' ' + (ConvertTo-SftpPath $remote))
    $batchFile = Join-Path $work 'sftp-batch.txt'
    Write-Utf8NoBom $batchFile (($lines -join "`r`n") + "`r`n")

    # BatchMode=no before -b: sftp -b turns BatchMode on (no password prompt),
    # and the first value given wins.
    $sftpArgs = @('-o', 'BatchMode=no', '-b', $batchFile, "$SshUser@$SshHost")
    Write-Step 'Upload'
    Write-Host "  sync.json:  $syncPath"
    Write-Host "  batch file: $batchFile"
    Write-Host "  command:    sftp $($sftpArgs -join ' ')"
    Write-Host ("  uploads {0} file(s), {1}, into {2}" -f $changed.Count, (Format-Size $uploadBytes), $remote)

    if ($DryRun) {
        Write-Host ''
        Write-Host 'Dry run: nothing uploaded. Run again without -DryRun to upload.' -ForegroundColor Yellow
        exit 0
    }

    $sftp = Get-Command sftp.exe -ErrorAction SilentlyContinue
    if (-not $sftp) { $sftp = Get-Command sftp -ErrorAction SilentlyContinue }
    if (-not $sftp) {
        throw 'sftp.exe not found (Settings > Apps > Optional features > OpenSSH Client). Or use -ZipOut and upload the zip on the webapp page.'
    }
    Write-Host ''
    Write-Host "Enter the password for $SshUser@$SshHost when asked." -ForegroundColor Cyan
    & $sftp.Path @sftpArgs
    if ($LASTEXITCODE -ne 0) {
        throw "sftp failed (exit $LASTEXITCODE). The partial upload is in $part on the server; delete it on the webapp page, then run this again."
    }

    Write-Host ''
    Write-Host "Uploaded batch $batch." -ForegroundColor Green
    Write-Host 'Validate and activate it here:'
    Write-Host ("  {0}batch/{1}" -f ($finishUrl.TrimEnd('/') + '/'), $batch)
    exit 0
} catch {
    Write-Host ''
    Write-Host "ERROR: $($_.Exception.Message)" -ForegroundColor Red
    Write-Host $_.ScriptStackTrace -ForegroundColor DarkGray
    exit 1
}