Windows installer: payload hash-locking, SBOM, client-IP handling, operator docs
This commit is contained in:
233
deploy/windows/installer/bundle-lock.ps1
Normal file
233
deploy/windows/installer/bundle-lock.ps1
Normal file
@@ -0,0 +1,233 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Hash manifest for the installer's third-party payload: create it, and check a
|
||||
bundle against it.
|
||||
|
||||
.DESCRIPTION
|
||||
Dot-source this. It defines two functions and runs nothing on its own:
|
||||
|
||||
New-BundleLock hash a staged bundle and return the lock object
|
||||
Test-BundleLock compare a staged bundle against a lock, return problems
|
||||
|
||||
WHAT IT COVERS, and why pip's own hash checking is not enough.
|
||||
|
||||
requirements.txt carries a sha256 for every wheel, so pip refuses an artifact
|
||||
upstream did not publish. Three gaps remain, and all three are what actually
|
||||
goes wrong with a hand-assembled offline bundle:
|
||||
|
||||
1. pip lists EVERY artifact of a pinned version - cffi 2.1.0 alone has 100
|
||||
hashes. It proves the wheel is genuine, not that it is the wheel this
|
||||
bundle was built and tested with.
|
||||
2. pip ignores extra files in the wheelhouse. A stale wheel left behind by
|
||||
a previous build sits there unnoticed until a resolve picks it up.
|
||||
3. pip says nothing about the rest of the payload - the Python installer,
|
||||
the HttpPlatformHandler MSI, URL Rewrite, MySQL. Those are executables
|
||||
that run as SYSTEM on the target server and were, until this file, the
|
||||
only unverified thing the installer would run.
|
||||
|
||||
So the lock records an exact file set with a sha256 and a byte size each, and
|
||||
verification is SET EQUALITY: a missing file, an unexpected extra file, or a
|
||||
changed file all fail. Nothing is skipped and nothing is "close enough".
|
||||
|
||||
The app tree is deliberately NOT covered. It is built from the repository on
|
||||
every run and changes with every commit; hashing it would make the lock churn
|
||||
constantly and train everyone to regenerate it without reading it. Git is the
|
||||
record for the app tree. This file is the record for everything that comes
|
||||
from outside the repository.
|
||||
|
||||
.NOTES
|
||||
Stock Windows PowerShell 5.1, and also runs under pwsh on Linux so the Bash
|
||||
builder can call the same checker.
|
||||
#>
|
||||
|
||||
# Payload directories under the bundle root. 'required' means the installer
|
||||
# cannot work without it; the optional ones are per-deployment choices, and an
|
||||
# absent optional directory is fine. A PRESENT directory is always checked in
|
||||
# full, optional or not.
|
||||
$script:BundlePayloads = @(
|
||||
@{ Name = 'wheels'; Required = $true; What = 'Python wheels for the offline install' }
|
||||
@{ Name = 'python'; Required = $true; What = 'the Python installer' }
|
||||
@{ Name = 'httpplatformhandler'; Required = $true; What = 'the IIS module that launches waitress' }
|
||||
@{ Name = 'urlrewrite'; Required = $false; What = 'IIS URL Rewrite, for the client-IP rule' }
|
||||
@{ Name = 'mysqlclient'; Required = $false; What = 'mysql/mysqldump, for backups against a remote database' }
|
||||
@{ Name = 'mysql'; Required = $false; What = 'MySQL, for the bundled-database option' }
|
||||
)
|
||||
|
||||
function Get-JsonProperty {
|
||||
# shopdb-install.ps1 runs under Set-StrictMode 2.0, where reading a property
|
||||
# that does not exist on a PSCustomObject THROWS instead of returning $null.
|
||||
# A truncated or hand-edited bundle-lock.json would therefore blow up with
|
||||
# "Property 'payloads' cannot be found" rather than saying what is wrong with
|
||||
# the lock. Every read of parsed JSON goes through here.
|
||||
param($Object, [string] $Name, $Default = $null)
|
||||
if ($null -eq $Object) { return $Default }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $Default }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Get-FileDigest {
|
||||
param([string] $Path)
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
$stream = [System.IO.File]::OpenRead($Path)
|
||||
try { return (-join ($sha.ComputeHash($stream) | ForEach-Object { $_.ToString('x2') })) }
|
||||
finally { $stream.Dispose(); $sha.Dispose() }
|
||||
}
|
||||
|
||||
function Get-PayloadFiles {
|
||||
# Every file in the directory, keyed by its path RELATIVE to that directory
|
||||
# with forward slashes, so a lock generated on Windows reads the same from
|
||||
# the Bash builder.
|
||||
param([string] $Dir)
|
||||
$out = @{}
|
||||
if (-not (Test-Path $Dir)) { return $out }
|
||||
$root = (Resolve-Path $Dir).Path.TrimEnd('\', '/')
|
||||
foreach ($f in (Get-ChildItem $Dir -Recurse -File)) {
|
||||
$rel = $f.FullName.Substring($root.Length).TrimStart('\', '/').Replace('\', '/')
|
||||
$out[$rel] = @{ sha256 = (Get-FileDigest $f.FullName); size = $f.Length }
|
||||
}
|
||||
return $out
|
||||
}
|
||||
|
||||
function New-BundleLock {
|
||||
<#
|
||||
Hash a staged bundle. The caller writes the result to bundle-lock.json;
|
||||
this returns the object so a caller can diff it against the committed lock
|
||||
before overwriting anything.
|
||||
#>
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string] $BundleRoot,
|
||||
[string] $PythonTag = 'cp314',
|
||||
[string] $Platform = 'win_amd64'
|
||||
)
|
||||
$payloads = @{}
|
||||
foreach ($p in $script:BundlePayloads) {
|
||||
$dir = Join-Path $BundleRoot $p.Name
|
||||
if (-not (Test-Path $dir)) {
|
||||
if ($p.Required) { throw ("payload directory is missing: {0} ({1})" -f $p.Name, $p.What) }
|
||||
continue
|
||||
}
|
||||
$files = Get-PayloadFiles $dir
|
||||
if ($files.Count -eq 0 -and $p.Required) {
|
||||
throw ("payload directory is empty: {0} ({1})" -f $p.Name, $p.What)
|
||||
}
|
||||
$payloads[$p.Name] = @{ required = $p.Required; files = $files }
|
||||
}
|
||||
return [ordered]@{
|
||||
schema = 1
|
||||
generated = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
|
||||
pythontag = $PythonTag
|
||||
platform = $Platform
|
||||
note = 'Exact third-party payload of the installer bundle. Regenerate with refresh-bundle-lock.ps1 and COMMIT the change as a reviewed dependency bump.'
|
||||
payloads = $payloads
|
||||
}
|
||||
}
|
||||
|
||||
function Test-BundleLock {
|
||||
<#
|
||||
Compare a staged bundle against a lock. Returns an array of problem
|
||||
strings - EMPTY means the bundle is exactly what the lock describes.
|
||||
|
||||
Returning problems rather than throwing is deliberate: an operator fixing a
|
||||
wheelhouse wants the whole list at once, not one failure per rebuild.
|
||||
#>
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string] $BundleRoot,
|
||||
[Parameter(Mandatory = $true)] $Lock
|
||||
)
|
||||
$problems = @()
|
||||
$payloads = Get-JsonProperty $Lock 'payloads'
|
||||
if ($null -eq $payloads) { return @('bundle-lock.json has no "payloads" section') }
|
||||
|
||||
foreach ($p in $script:BundlePayloads) {
|
||||
$name = $p.Name
|
||||
$dir = Join-Path $BundleRoot $name
|
||||
$present = Test-Path $dir
|
||||
$entry = Get-JsonProperty $payloads $name
|
||||
$locked = ($null -ne $entry)
|
||||
|
||||
if (-not $locked) {
|
||||
# Not in the lock at all. An unlocked directory that exists is a
|
||||
# payload nobody reviewed, which is exactly what this is here to stop.
|
||||
if ($present) { $problems += "$name/ is present but is not in bundle-lock.json - regenerate the lock" }
|
||||
elseif ($p.Required) { $problems += "$name/ is required but is in neither the bundle nor the lock" }
|
||||
continue
|
||||
}
|
||||
if (-not $present) {
|
||||
if ($p.Required -or (Get-JsonProperty $entry 'required' $false)) {
|
||||
$problems += "$name/ is in the lock but missing from the bundle ($($p.What))"
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
$expected = @{}
|
||||
$lockedFiles = Get-JsonProperty $entry 'files'
|
||||
if ($null -ne $lockedFiles) {
|
||||
foreach ($prop in $lockedFiles.PSObject.Properties) { $expected[$prop.Name] = $prop.Value }
|
||||
}
|
||||
$actual = Get-PayloadFiles $dir
|
||||
|
||||
# Sorted so the report reads the same way twice, and so it matches the
|
||||
# order verify_bundle_lock.py produces.
|
||||
foreach ($rel in ($expected.Keys | Sort-Object)) {
|
||||
if (-not $actual.ContainsKey($rel)) { $problems += "$name/$rel is in the lock but missing from the bundle"; continue }
|
||||
if ($actual[$rel].sha256 -ne $expected[$rel].sha256) {
|
||||
$problems += "$name/$rel does NOT match the lock (expected sha256 $($expected[$rel].sha256.Substring(0,12))..., got $($actual[$rel].sha256.Substring(0,12))...)"
|
||||
} elseif ([int64] $actual[$rel].size -ne [int64] $expected[$rel].size) {
|
||||
# Cannot happen for a matching sha256, so it means the lock itself
|
||||
# was hand-edited. Say so rather than passing it.
|
||||
$problems += "$name/$rel size disagrees with the lock - the lock has been edited by hand"
|
||||
}
|
||||
}
|
||||
foreach ($rel in ($actual.Keys | Sort-Object)) {
|
||||
if (-not $expected.ContainsKey($rel)) { $problems += "$name/$rel is in the bundle but NOT in the lock (unexpected extra file)" }
|
||||
}
|
||||
}
|
||||
$problems += Test-WheelhouseCoversRequirements -BundleRoot $BundleRoot
|
||||
return $problems
|
||||
}
|
||||
|
||||
function Test-WheelhouseCoversRequirements {
|
||||
<#
|
||||
The lock records what IS in the wheelhouse, not what the application NEEDS.
|
||||
Without this an incomplete wheelhouse gets locked, blessed, and shipped,
|
||||
and the install fails on an air-gapped server.
|
||||
|
||||
Not hypothetical: assembling the wheelhouse anywhere other than Windows
|
||||
silently omits colorama, a win32-only dependency of click, because pip
|
||||
evaluates environment markers against the machine doing the downloading
|
||||
rather than the machine being targeted. Markers are therefore IGNORED here
|
||||
- a requirement guarded by sys_platform == 'win32' is precisely the one
|
||||
that has to be present.
|
||||
#>
|
||||
param([Parameter(Mandatory = $true)] [string] $BundleRoot)
|
||||
$wheels = Join-Path $BundleRoot 'wheels'
|
||||
$reqs = Join-Path $BundleRoot 'app\requirements.txt'
|
||||
if (-not (Test-Path $wheels) -or -not (Test-Path $reqs)) { return @() }
|
||||
|
||||
$have = @(Get-ChildItem $wheels -File -ErrorAction SilentlyContinue | ForEach-Object { $_.Name.ToLower() })
|
||||
$problems = @()
|
||||
$pins = @{}
|
||||
foreach ($line in (Get-Content $reqs)) {
|
||||
$trimmed = $line.Trim()
|
||||
if (-not $trimmed -or $trimmed.StartsWith('#')) { continue }
|
||||
if ($trimmed -match '^([A-Za-z0-9._-]+)==([^\s;\\]+)') {
|
||||
# PEP 427 wheel filename form: runs of non-alphanumerics become one _.
|
||||
$pins[([regex]::Replace($Matches[1], '[^A-Za-z0-9.]+', '_')).ToLower()] = $Matches[2]
|
||||
}
|
||||
}
|
||||
foreach ($name in ($pins.Keys | Sort-Object)) {
|
||||
$prefix = "$name-$($pins[$name])-"
|
||||
if (-not ($have | Where-Object { $_.StartsWith($prefix) })) {
|
||||
$problems += ("wheels/ has no wheel for {0}=={1}, which requirements.txt pins " +
|
||||
"(a marked-out dependency still installs on Windows)") -f $name, $pins[$name]
|
||||
}
|
||||
}
|
||||
return $problems
|
||||
}
|
||||
|
||||
function Read-BundleLock {
|
||||
param([Parameter(Mandatory = $true)] [string] $Path)
|
||||
if (-not (Test-Path $Path)) { return $null }
|
||||
return (Get-Content $Path -Raw | ConvertFrom-Json)
|
||||
}
|
||||
Reference in New Issue
Block a user