<# .SYNOPSIS Hash manifest for the installer's third-party payload: create it, and check a bundle against it. .DESCRIPTION Dot-source this. It defines two functions and runs nothing on its own: New-BundleLock hash a staged bundle and return the lock object Test-BundleLock compare a staged bundle against a lock, return problems WHAT IT COVERS, and why pip's own hash checking is not enough. requirements.txt carries a sha256 for every wheel, so pip refuses an artifact upstream did not publish. Three gaps remain, and all three are what actually goes wrong with a hand-assembled offline bundle: 1. pip lists EVERY artifact of a pinned version - cffi 2.1.0 alone has 100 hashes. It proves the wheel is genuine, not that it is the wheel this bundle was built and tested with. 2. pip ignores extra files in the wheelhouse. A stale wheel left behind by a previous build sits there unnoticed until a resolve picks it up. 3. pip says nothing about the rest of the payload - the Python installer, the HttpPlatformHandler MSI, URL Rewrite, MySQL. Those are executables that run as SYSTEM on the target server and were, until this file, the only unverified thing the installer would run. So the lock records an exact file set with a sha256 and a byte size each, and verification is SET EQUALITY: a missing file, an unexpected extra file, or a changed file all fail. Nothing is skipped and nothing is "close enough". The app tree is deliberately NOT covered. It is built from the repository on every run and changes with every commit; hashing it would make the lock churn constantly and train everyone to regenerate it without reading it. Git is the record for the app tree. This file is the record for everything that comes from outside the repository. .NOTES Stock Windows PowerShell 5.1, and also runs under pwsh on Linux so the Bash builder can call the same checker. #> # Payload directories under the bundle root. 'required' means the installer # cannot work without it; the optional ones are per-deployment choices, and an # absent optional directory is fine. A PRESENT directory is always checked in # full, optional or not. $script:BundlePayloads = @( @{ Name = 'wheels'; Required = $true; What = 'Python wheels for the offline install' } @{ Name = 'python'; Required = $true; What = 'the Python installer' } @{ Name = 'httpplatformhandler'; Required = $true; What = 'the IIS module that launches waitress' } @{ Name = 'urlrewrite'; Required = $false; What = 'IIS URL Rewrite, for the client-IP rule' } @{ Name = 'mysql'; Required = $false; What = 'MySQL, for the bundled-database option' } ) function Get-JsonProperty { # shopdb-install.ps1 runs under Set-StrictMode 2.0, where reading a property # that does not exist on a PSCustomObject THROWS instead of returning $null. # A truncated or hand-edited bundle-lock.json would therefore blow up with # "Property 'payloads' cannot be found" rather than saying what is wrong with # the lock. Every read of parsed JSON goes through here. param($Object, [string] $Name, $Default = $null) if ($null -eq $Object) { return $Default } $prop = $Object.PSObject.Properties[$Name] if ($null -eq $prop) { return $Default } return $prop.Value } function Get-FileDigest { param([string] $Path) $sha = [System.Security.Cryptography.SHA256]::Create() $stream = [System.IO.File]::OpenRead($Path) try { return (-join ($sha.ComputeHash($stream) | ForEach-Object { $_.ToString('x2') })) } finally { $stream.Dispose(); $sha.Dispose() } } function Get-PayloadFiles { # Every file in the directory, keyed by its path RELATIVE to that directory # with forward slashes, so a lock generated on Windows reads the same from # the Bash builder. param([string] $Dir) $out = @{} if (-not (Test-Path $Dir)) { return $out } $root = (Resolve-Path $Dir).Path.TrimEnd('\', '/') foreach ($f in (Get-ChildItem $Dir -Recurse -File)) { $rel = $f.FullName.Substring($root.Length).TrimStart('\', '/').Replace('\', '/') $out[$rel] = @{ sha256 = (Get-FileDigest $f.FullName); size = $f.Length } } return $out } function New-BundleLock { <# Hash a staged bundle. The caller writes the result to bundle-lock.json; this returns the object so a caller can diff it against the committed lock before overwriting anything. #> param( [Parameter(Mandatory = $true)] [string] $BundleRoot, [string] $PythonTag = 'cp314', [string] $Platform = 'win_amd64' ) $payloads = @{} foreach ($p in $script:BundlePayloads) { $dir = Join-Path $BundleRoot $p.Name if (-not (Test-Path $dir)) { if ($p.Required) { throw ("payload directory is missing: {0} ({1})" -f $p.Name, $p.What) } continue } $files = Get-PayloadFiles $dir if ($files.Count -eq 0 -and $p.Required) { throw ("payload directory is empty: {0} ({1})" -f $p.Name, $p.What) } $payloads[$p.Name] = @{ required = $p.Required; files = $files } } return [ordered]@{ schema = 1 generated = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') pythontag = $PythonTag platform = $Platform note = 'Exact third-party payload of the installer bundle. Regenerate with refresh-bundle-lock.ps1 and COMMIT the change as a reviewed dependency bump.' payloads = $payloads } } function Test-BundleLock { <# Compare a staged bundle against a lock. Returns an array of problem strings - EMPTY means the bundle is exactly what the lock describes. Returning problems rather than throwing is deliberate: an operator fixing a wheelhouse wants the whole list at once, not one failure per rebuild. #> param( [Parameter(Mandatory = $true)] [string] $BundleRoot, [Parameter(Mandatory = $true)] $Lock ) $problems = @() $payloads = Get-JsonProperty $Lock 'payloads' if ($null -eq $payloads) { return @('bundle-lock.json has no "payloads" section') } foreach ($p in $script:BundlePayloads) { $name = $p.Name $dir = Join-Path $BundleRoot $name $present = Test-Path $dir $entry = Get-JsonProperty $payloads $name $locked = ($null -ne $entry) if (-not $locked) { # Not in the lock at all. An unlocked directory that exists is a # payload nobody reviewed, which is exactly what this is here to stop. if ($present) { $problems += "$name/ is present but is not in bundle-lock.json - regenerate the lock" } elseif ($p.Required) { $problems += "$name/ is required but is in neither the bundle nor the lock" } continue } if (-not $present) { if ($p.Required -or (Get-JsonProperty $entry 'required' $false)) { $problems += "$name/ is in the lock but missing from the bundle ($($p.What))" } continue } $expected = @{} $lockedFiles = Get-JsonProperty $entry 'files' if ($null -ne $lockedFiles) { foreach ($prop in $lockedFiles.PSObject.Properties) { $expected[$prop.Name] = $prop.Value } } $actual = Get-PayloadFiles $dir # Sorted so the report reads the same way twice, and so it matches the # order verify_bundle_lock.py produces. foreach ($rel in ($expected.Keys | Sort-Object)) { if (-not $actual.ContainsKey($rel)) { $problems += "$name/$rel is in the lock but missing from the bundle"; continue } if ($actual[$rel].sha256 -ne $expected[$rel].sha256) { $problems += "$name/$rel does NOT match the lock (expected sha256 $($expected[$rel].sha256.Substring(0,12))..., got $($actual[$rel].sha256.Substring(0,12))...)" } elseif ([int64] $actual[$rel].size -ne [int64] $expected[$rel].size) { # Cannot happen for a matching sha256, so it means the lock itself # was hand-edited. Say so rather than passing it. $problems += "$name/$rel size disagrees with the lock - the lock has been edited by hand" } } foreach ($rel in ($actual.Keys | Sort-Object)) { if (-not $expected.ContainsKey($rel)) { $problems += "$name/$rel is in the bundle but NOT in the lock (unexpected extra file)" } } } $problems += Test-WheelhouseCoversRequirements -BundleRoot $BundleRoot return $problems } function Test-WheelhouseCoversRequirements { <# The lock records what IS in the wheelhouse, not what the application NEEDS. Without this an incomplete wheelhouse gets locked, blessed, and shipped, and the install fails on an air-gapped server. Not hypothetical: assembling the wheelhouse anywhere other than Windows silently omits colorama, a win32-only dependency of click, because pip evaluates environment markers against the machine doing the downloading rather than the machine being targeted. Markers are therefore IGNORED here - a requirement guarded by sys_platform == 'win32' is precisely the one that has to be present. #> param([Parameter(Mandatory = $true)] [string] $BundleRoot) $wheels = Join-Path $BundleRoot 'wheels' $reqs = Join-Path $BundleRoot 'app\requirements.txt' if (-not (Test-Path $wheels) -or -not (Test-Path $reqs)) { return @() } $have = @(Get-ChildItem $wheels -File -ErrorAction SilentlyContinue | ForEach-Object { $_.Name.ToLower() }) $problems = @() $pins = @{} foreach ($line in (Get-Content $reqs)) { $trimmed = $line.Trim() if (-not $trimmed -or $trimmed.StartsWith('#')) { continue } if ($trimmed -match '^([A-Za-z0-9._-]+)==([^\s;\\]+)') { # PEP 427 wheel filename form: runs of non-alphanumerics become one _. $pins[([regex]::Replace($Matches[1], '[^A-Za-z0-9.]+', '_')).ToLower()] = $Matches[2] } } foreach ($name in ($pins.Keys | Sort-Object)) { $prefix = "$name-$($pins[$name])-" if (-not ($have | Where-Object { $_.StartsWith($prefix) })) { $problems += ("wheels/ has no wheel for {0}=={1}, which requirements.txt pins " + "(a marked-out dependency still installs on Windows)") -f $name, $pins[$name] } } return $problems } function Read-BundleLock { param([Parameter(Mandatory = $true)] [string] $Path) if (-not (Test-Path $Path)) { return $null } return (Get-Content $Path -Raw | ConvertFrom-Json) }