From d8e3280dfc79ecbb7a1aa9f6dcee6253da4203a3 Mon Sep 17 00:00:00 2001 From: Cam Proudlock Date: Wed, 5 Aug 2026 07:36:43 -0400 Subject: [PATCH] Release 0.8.0: Windows installer, twelve fixes from a real Server 2019 install, operator docs --- CHANGELOG.md | 66 +- deploy/site-profile-universal.json | 20 + deploy/windows/installer/README.md | 6 +- deploy/windows/installer/ShopDBFlask.iss | 211 ++++- deploy/windows/installer/build-installer.ps1 | 2 +- deploy/windows/installer/build-installer.sh | 5 +- deploy/windows/installer/bundle-lock.json | 411 +++++---- deploy/windows/installer/bundle-lock.ps1 | 42 +- deploy/windows/installer/shopdb-admin.ps1 | 57 +- deploy/windows/installer/shopdb-install.ps1 | 811 +++++++++++++++--- deploy/windows/installer/shopdb-preflight.ps1 | 31 +- .../installer/tests/test-install-state.ps1 | 130 +++ .../windows/installer/verify_bundle_lock.py | 1 + deploy/windows/shopdb-diagnose.py | 422 +++++++++ deploy/windows/web.config | 23 +- frontend/package.json | 2 +- pytest.ini | 28 + requirements.in | 6 + requirements.txt | 77 +- scripts/gen_openapi.py | 20 +- shopdb/__init__.py | 5 +- shopdb/cli/__init__.py | 157 ++++ shopdb/core/services/csvimport.py | 500 +++++++++++ tests/test_csv_import.py | 236 +++++ tests/test_installer_defaults.py | 51 ++ tests/test_installer_state.py | 106 +++ tests/test_runtime_dependencies.py | 159 ++++ 27 files changed, 3189 insertions(+), 396 deletions(-) create mode 100644 deploy/site-profile-universal.json create mode 100644 deploy/windows/installer/tests/test-install-state.ps1 create mode 100644 deploy/windows/shopdb-diagnose.py create mode 100644 pytest.ini create mode 100644 shopdb/core/services/csvimport.py create mode 100644 tests/test_csv_import.py create mode 100644 tests/test_installer_state.py create mode 100644 tests/test_runtime_dependencies.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 721e9aa6..398e4540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,71 @@ ADR-007 and ADR-002. ## [Unreleased] -### Added +## [0.8.0] - 2026-08-05 + +First release to carry the Windows installer. Everything below shipped after +v0.7.0 was tagged, and a complete install was exercised end to end on Windows +Server 2019 before this release was cut. + +### Fixed - installer, from a real Server 2019 install + +- `packaging` was imported by the plugin loader but declared nowhere. It reached + development and CI only as a dependency of pytest, so the whole suite passed + while a virtual environment built from `requirements.txt` alone - which is + exactly what the installer builds - could not import the application at all. + `tests/test_runtime_dependencies.py` now fails on any runtime import that is + not a declared dependency. +- IIS returned 500.52 before the application was ever launched. `web.config` + declared `` for the X-Forwarded-For rule, and that + section ships `overrideModeDefault="Deny"`, so the whole file was rejected. + The installer now permits the single variable at server level instead of + unlocking the section for every site on the machine, and repairs a + `web.config` an earlier build had made unusable. +- The config unlock ran before the application it unlocks existed, so the scoped + form could never succeed on a first install and every install silently fell + back to granting handler delegation server-wide. +- The stage 5 smoke test discarded the status code and error page it had already + received, reporting "site did not return 200" for a fault IIS had named. It + now records both, plus the tail of the application log. +- Database dumps were readable by every authenticated user: a directory created + under ProgramData inherits `Users:RX`, and the owner-only ACL was applied only + when the installer itself created it. +- The uninstaller ran the 32-bit PowerShell, which cannot see IIS, so the site, + pool and application survived a "successful" uninstall pointing at a deleted + directory. +- Uninstall matched applications by alias alone and would remove an unrelated + application of the same name under another site. +- Wizard input reached a command line unchecked: unvalidated ports, a drive-root + path that escaped its own quote, and a password written as ANSI but read back + as UTF-8, which reported a correct non-ASCII password as wrong. +- Plugin deregistration could never succeed - it omitted `--yes` against a + command that prompts - while the plugin directory was deleted regardless. +- Preflight rows were drawn past the bottom of the panel and silently vanished; + the failures loop had no cap at all. + +### Changed - installer behaviour + +- "Is this a re-run of my install?" is answered from a durable install record + rather than inferred from the state of the machine. Nothing on a server says + who created its database tables, so a retry after a failed first install was + taken for an upgrade of a working system: it demanded a mandatory backup of a + database its own failed attempt had written, then refused to prune tables it + had created minutes earlier. During unfinished first provisioning the backup + is advisory and prune may force; on an established install both are unchanged. +- The installer offers every bundled plugin, so one build serves any site + instead of one build per plugin profile. + +### Added - operator documentation and diagnostics + +- `docs/UPDATES-WINDOWS.md` - what operators should expect from future updates, + bug fixes and security releases, including downtime, what is preserved, and + the effect on other sites sharing the same IIS server. +- `docs/RELEASING-WINDOWS.md` - how to build and release, and the two known gaps. +- `deploy/windows/shopdb-diagnose.py` - collects what IIS answers, the config + lock state, the application logs and the ACLs in one pass, scrubbing secrets + before writing anything. + +### Added - the installer itself - Air-gapped Windows installer (`deploy/windows/installer/`). One `.exe` per site, built from that site's plugin profile, containing Python, the wheels, diff --git a/deploy/site-profile-universal.json b/deploy/site-profile-universal.json new file mode 100644 index 00000000..a80a16ea --- /dev/null +++ b/deploy/site-profile-universal.json @@ -0,0 +1,20 @@ +{ + "site": "universal", + "plugins": [ + "computers", + "employees", + "geenforce", + "knowledgebase", + "machines", + "measuringtools", + "network", + "notifications", + "printedparts", + "printers", + "slides", + "usb", + "warranty" + ], + "locked": [], + "_comment": "The profile the released Windows installer is built from. Every bundled plugin that carries a manifest, so ONE exe serves any site: the wizard offers all of them and the operator ticks what that site uses. Plugins left unticked are never installed, and 'flask plugin prune-schema' drops their tables at provisioning (ADR-014). 'applications' is deliberately absent - it is manifest-less core and always ships. Build with: deploy/windows/installer/build-installer.sh deploy/site-profile-universal.json . Use site-profile.example.json instead only when a site genuinely needs a lean build; see ADR-013." +} diff --git a/deploy/windows/installer/README.md b/deploy/windows/installer/README.md index 7dbf084d..0e50dc4b 100644 --- a/deploy/windows/installer/README.md +++ b/deploy/windows/installer/README.md @@ -53,7 +53,9 @@ bundle/wheels/ pip download -r requirements.txt --only-binary=:all bundle/python/ python-3.14.x-amd64.exe bundle/httpplatformhandler/ httpPlatformHandler_amd64.msi bundle/urlrewrite/ rewrite_amd64.msi (client-IP rule; see below) -bundle/mysql/ mysql-8.0.x-winx64.msi (bundled-database option only) +bundle/vcredist/ VC_redist.x64.exe (MySQL will not install without it) +bundle/mysql/ mysql-8.4.x-winx64.msi (bundled-database option only) +bundle/mysqlclient/ mysql.exe, mysqldump.exe + libcrypto/libssl (backups) ``` Then compile on Windows: @@ -235,7 +237,7 @@ database is an upgrade, and treating it as fresh would drop tables. ## Testing notes -Verified end to end on Windows Server 2025 against both a bundled MySQL 8.0 and +Verified end to end on Windows Server 2025 against both a bundled MySQL 8.4 LTS and an existing MySQL 5.6: fresh install, upgrade, re-run idempotency, failure and rollback, uninstall, and both deployment methods including switching between them. diff --git a/deploy/windows/installer/ShopDBFlask.iss b/deploy/windows/installer/ShopDBFlask.iss index 7bbecd29..e39dbfe6 100644 --- a/deploy/windows/installer/ShopDBFlask.iss +++ b/deploy/windows/installer/ShopDBFlask.iss @@ -4,7 +4,7 @@ ; ; DESIGN NOTE - why this is a thin wrapper and not a reimplementation: ; every install action lives in shopdb-install.ps1, which is verified end to end -; on Windows Server 2025 against both a bundled MySQL 8.0 and an existing MySQL +; on Windows Server 2025 against both a bundled MySQL 8.4 LTS and an existing MySQL ; 5.6. Reimplementing any of it in Pascal Script would create a second code path ; that nobody tests. This file only collects operator input, runs the stages in ; order, and reports which one failed. @@ -181,6 +181,7 @@ Filename: "{win}\Sysnative\WindowsPowerShell\v1.0\powershell.exe"; \ var DbChoicePage: TInputOptionWizardPage; DbDetailsPage: TInputQueryWizardPage; + DbCredsPage: TInputQueryWizardPage; SitePage: TInputQueryWizardPage; PreflightPage: TWizardPage; BannerPanel: TPanel; @@ -402,33 +403,40 @@ begin 'choose the second option - installing a second server would collide on ' + 'port 3306.', True, False); - DbChoicePage.Add('Install the bundled MySQL 8.0 (new servers only)'); + DbChoicePage.Add('Install the bundled MySQL 8.4 LTS (new servers only)'); DbChoicePage.Add('Use a MySQL server this site already runs'); DbChoicePage.SelectedValueIndex := DB_EXISTING; DbDetailsPage := CreateInputQueryPage(DbChoicePage.ID, 'Existing database', 'Connection details', - 'The database and application user must already exist. Your DBA creates ' + - 'them; the installer does not, so that it never needs administrative ' + - 'rights on your database server. If they need the exact statements:' + #13#10 + - ' CREATE DATABASE shopdb_flask CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;' + #13#10 + - ' CREATE USER ''shopdb''@''%'' IDENTIFIED BY '''';' + #13#10 + - ' GRANT ALL PRIVILEGES ON shopdb_flask.* TO ''shopdb''@''%'';' + #13#10 + - 'The utf8mb4 charset matters - the default on older servers is latin1.'); + 'The database and application user must already exist - the installer does ' + + 'not create them, so it never needs administrative rights on your database ' + + 'server. The exact SQL to give your DBA is in the INSTALL-WINDOWS guide.'); DbDetailsPage.Add('Host:', False); DbDetailsPage.Add('Port:', False); DbDetailsPage.Add('Database:', False); - DbDetailsPage.Add('Username:', False); - DbDetailsPage.Add('Password:', True); DbDetailsPage.Values[0] := '127.0.0.1'; DbDetailsPage.Values[1] := '3306'; DbDetailsPage.Values[2] := 'shopdb_flask'; - DbDetailsPage.Values[3] := 'shopdb'; + + // SPLIT ACROSS TWO PAGES on purpose. CreateInputQueryPage stacks its fields + // below the description and does not scroll or shrink: with five fields the + // last one is drawn past the bottom of the surface and simply does not appear. + // Trimming the description bought one field back and lost the next, which is + // guessing at a pixel budget that varies with DPI and font. Three fields and + // two fields both fit under any reasonable description, at any scaling. + DbCredsPage := CreateInputQueryPage(DbDetailsPage.ID, + 'Existing database', 'Sign-in', + 'The account the application uses to reach that database. It needs full ' + + 'rights on it, and nothing outside it.'); + DbCredsPage.Add('Username:', False); + DbCredsPage.Add('Password:', True); + DbCredsPage.Values[0] := 'shopdb'; // How the application is published. Offered only when the bundle actually // carries a subpath SPA build - Vite compiles the base path in, so this can // never be a pure runtime switch. - DeployPage := CreateInputOptionPage(DbDetailsPage.ID, + DeployPage := CreateInputOptionPage(DbCredsPage.ID, 'Address', 'How should people reach ShopDB-Flask?', 'Both options serve the same application. The second needs no new DNS name ' + 'and no port number, because it rides this server''s existing address.', @@ -647,7 +655,7 @@ end; // If nothing is wrong the page collapses to a single reassuring statement. procedure RenderPreflight(Lines: TArrayOfString; var HasBlockers: Boolean); var - I, Y, Shown, Fails, Warns, Passes: Integer; + I, Y, Shown, Fails, Warns, Passes, RowHeight, MaxRows, Hidden: Integer; Parts: TArrayOfString; Check, Detail, Fix: String; begin @@ -655,6 +663,16 @@ begin Y := ScaleY(4); Shown := 0; + // How many rows actually FIT, measured, not guessed. The old cap of 6 bore no + // relation to the panel, which holds roughly three: at four or five notes the + // rows below were drawn past the bottom edge and silently vanished - taking + // with them the warning that the bundled MySQL collides on port 3306, which is + // exactly what the operator needs before the very next page. + RowHeight := ScaleY(46) + ScaleY(ROW_GAP); + if RowHeight < 1 then RowHeight := 1; + MaxRows := DetailPanel.Height div RowHeight; + if MaxRows < 1 then MaxRows := 1; + // Blockers first, then warnings - a second pass rather than one, so severity // ordering does not depend on the order the checks happen to run in. for I := 0 to GetArrayLength(Lines) - 1 do @@ -668,6 +686,9 @@ begin for I := 0 to GetArrayLength(Lines) - 1 do begin + // Blockers are capped too. Without this the FAIL loop drew every failure, + // however many, straight off the bottom of the panel. + if Shown >= MaxRows then Break; Parts := StringSplit(Lines[I], ['|'], stAll); if GetArrayLength(Parts) < 4 then Continue; if Parts[0] <> 'FAIL' then Continue; @@ -682,7 +703,7 @@ begin for I := 0 to GetArrayLength(Lines) - 1 do begin - if Shown >= 6 then Break; + if Shown >= MaxRows then Break; Parts := StringSplit(Lines[I], ['|'], stAll); if GetArrayLength(Parts) < 4 then Continue; if Parts[0] <> 'WARN' then Continue; @@ -719,8 +740,13 @@ begin + 'verified. Nothing needs your attention.'); end; - if (Shown >= 6) and (Warns > 6 - Fails) then - FooterText.Caption := 'Some notes are not shown. The full check is in the install log.' + // Count what was actually left out rather than inferring it from the old cap, + // which under-reported: with 2 failures and 5 warnings it claimed everything + // was shown while four rows had been dropped. + Hidden := (Fails + Warns) - Shown; + if Hidden > 0 then + FooterText.Caption := IntToStr(Hidden) + ' further item(s) are not shown here. ' + + 'All of them are in the install log.' else FooterText.Caption := IntToStr(Passes) + ' checks passed. Nothing has been changed on this server.'; end; @@ -829,10 +855,76 @@ begin Log('[shopdb] pre-fill base directory: ' + Result); end; +// Everything typed in the wizard ends up on a command line that +// CommandLineToArgvW parses, and reaches PowerShell parameters typed [int] or +// [string]. A blank, a space, or a trailing backslash therefore has to be caught +// HERE - by the time the script sees it, the arguments have already shifted. + +function IsValidPort(const S: String): Boolean; +var + I, N: Integer; +begin + Result := False; + if (S = '') or (Length(S) > 5) then Exit; + for I := 1 to Length(S) do + if (S[I] < '0') or (S[I] > '9') then Exit; // also rejects spaces and signs + N := StrToIntDef(S, -1); + Result := (N >= 1) and (N <= 65535); +end; + +function IsAsciiOnly(const S: String): Boolean; +var + I: Integer; +begin + Result := True; + for I := 1 to Length(S) do + if Ord(S[I]) > 126 then + begin + Result := False; + Exit; + end; +end; + +// A path that ends in a backslash - a drive root such as D:\ - closes the +// argument with \" , which CommandLineToArgvW reads as an ESCAPED quote. The +// argument never terminates and every argument after it shifts one place. +// Doubling the backslash inside the quotes is the documented way out. +function QuotePathArg(const P: String): String; +begin + if (P <> '') and (P[Length(P)] = '\') then + Result := '"' + P + '\' + '"' + else + Result := '"' + P + '"'; +end; + function NextButtonClick(CurPageID: Integer): Boolean; begin Result := True; + // A drive root is refused outright rather than quoted around. Uninstall + // removes the application directory recursively, so accepting D:\ here would + // mean uninstalling ShopDB-Flask wipes the whole drive. + if CurPageID = wpSelectDir then + begin + if Length(WizardDirValue) <= 3 then + begin + MsgBox('Choose a folder rather than a whole drive.' + #13#10#13#10 + + 'Removing ShopDB-Flask deletes the folder it was installed into, ' + + 'so installing to a drive root would delete everything on that ' + + 'drive when it is uninstalled.', mbError, MB_OK); + Result := False; + end; + end; + + if CurPageID = SitePage.ID then + begin + if not IsValidPort(SitePage.Values[1]) then + begin + MsgBox('Enter the port as a number between 1 and 65535.', mbError, MB_OK); + Result := False; + end; + end; + if CurPageID = wpWelcome then RunPreflight; @@ -865,8 +957,34 @@ begin MsgBox('Enter the database host.', mbError, MB_OK); Result := False; end - else if (DbDetailsPage.Values[4] = '') and - (not FileExists(InstalledDir + '.env')) then + else if not IsValidPort(DbDetailsPage.Values[1]) then + begin + MsgBox('Enter the database port as a number between 1 and 65535.' + + #13#10#13#10 + 'MySQL uses 3306 unless it was changed.', + mbError, MB_OK); + Result := False; + end; + end; + end; + + if CurPageID = DbCredsPage.ID then + begin + if not UseBundledDb then + begin + if DbCredsPage.Values[0] = '' then + begin + MsgBox('Enter the username the application connects with.', mbError, MB_OK); + Result := False; + end + // Blank is allowed when the password is already ON DISK in a form the + // installer can read: .env from a previous install, or the ACL'd .dbpass + // handoff stage 0 leaves when it created the database itself. Stage 2 + // consumes .dbpass on its own, so demanding the password here blocked the + // operator on something the installer already had - and sent them looking + // for a generated secret they were never meant to handle. + else if (DbCredsPage.Values[1] = '') and + (not FileExists(InstalledDir + '.env')) and + (not FileExists(InstalledDir + '.dbpass')) then begin // Only required on a FRESH install. On an upgrade, blank means "keep the // password already in .env", so the operator never has to know it. @@ -913,6 +1031,13 @@ begin if (CurPageID = DbDetailsPage.ID) and (not DbPageReady) then begin DbPageReady := True; + // A stage-0 handoff means the database was created by a previous run of this + // installer and the password is already on disk. Nobody should be asked to + // find or retype a secret the installer generated. + if FileExists(InstalledDir + '.dbpass') then + DbCredsPage.SubCaptionLabel.Caption := + 'This server already has a database created by this installer. Leave the ' + + 'password blank and it will be used automatically.'; if LoadStringsFromFile(InstalledDir + '.env', Lines) then for I := 0 to GetArrayLength(Lines) - 1 do if Pos('DATABASE_URL=', Lines[I]) = 1 then @@ -924,7 +1049,7 @@ begin Creds := Copy(Existing, 1, Pos('@', Existing) - 1); Rest := Copy(Existing, Pos('@', Existing) + 1, Length(Existing)); if Pos(':', Creds) > 0 then - DbDetailsPage.Values[3] := Copy(Creds, 1, Pos(':', Creds) - 1); + DbCredsPage.Values[0] := Copy(Creds, 1, Pos(':', Creds) - 1); if Pos(':', Rest) > 0 then begin DbDetailsPage.Values[0] := Copy(Rest, 1, Pos(':', Rest) - 1); @@ -940,10 +1065,13 @@ begin end; // Password intentionally left blank: blank means "keep the current // one", so an upgrade never needs the operator to know it. - DbDetailsPage.Values[4] := ''; + DbCredsPage.Values[1] := ''; DbDetailsPage.SubCaptionLabel.Caption := - 'These are the settings this server is using now. Leave the password ' - + 'blank to keep the current one.'; + 'These are the settings this server is using now.'; + // The hint belongs where the password field actually is - it moved to + // the next page when this one was split. + DbCredsPage.SubCaptionLabel.Caption := + 'Leave the password blank to keep the one this server already uses.'; end; end; @@ -994,7 +1122,7 @@ begin end; // The bundled path generates its own credentials, so asking for them would be // meaningless - and any value typed here would be silently ignored. - Result := (PageID = DbDetailsPage.ID) and UseBundledDb; + Result := ((PageID = DbDetailsPage.ID) or (PageID = DbCredsPage.ID)) and UseBundledDb; end; // Runs from ssPostInstall, NOT PrepareToInstall. @@ -1019,22 +1147,31 @@ var Cmd: String; begin if not FileExists(LnkPath) then Exit; + // THREE quotes each side, not four. Four is an escaped quote twice over, which + // keeps the whole expression inside one Pascal literal: LnkPath was never + // interpolated, PowerShell got the bare word "LnkPath", failed to parse, and + // no shortcut ever received the elevation flag. Self-elevation hid it - the + // only symptom was a second window after an extra prompt. Cmd := '-NoProfile -ExecutionPolicy Bypass -Command "' - + '$p='''' + LnkPath + ''''; ' + + '$p=''' + LnkPath + '''; ' + '$b=[IO.File]::ReadAllBytes($p); ' + '$b[21]=$b[21] -bor 0x20; ' + '[IO.File]::WriteAllBytes($p,$b)"'; Exec(PowerShellPath, Cmd, '', SW_HIDE, ewWaitUntilTerminated, ResultCode); + if ResultCode <> 0 then + Log('[shopdb] could not mark ' + LnkPath + ' run-as-administrator (exit ' + + IntToStr(ResultCode) + '); the console will self-elevate instead'); end; function RunInstallStages: String; var ResultCode: Integer; PwFile, Args, Common: String; + PwLines: TArrayOfString; begin Result := ''; - Common := '-BundleRoot "' + ExpandConstant('{tmp}\shopdb-bundle') + '"' + - ' -AppRoot "' + ExpandConstant('{app}') + '"' + + Common := '-BundleRoot ' + QuotePathArg(ExpandConstant('{tmp}\shopdb-bundle')) + + ' -AppRoot ' + QuotePathArg(ExpandConstant('{app}')) + ' -SiteHost "' + SitePage.Values[0] + '"' + ' -SitePort ' + SitePage.Values[1] + ' -OnFailure never' + @@ -1070,10 +1207,16 @@ begin // PowerShell transcripts. The installer shreds the file after reading it. // No password typed on an upgrade means "leave .env alone" - so send no // password file, and the installer keeps the existing DATABASE_URL. - if DbDetailsPage.Values[4] <> '' then + if DbCredsPage.Values[1] <> '' then begin PwFile := ExpandConstant('{tmp}\dbpw.txt'); - SaveStringToFile(PwFile, DbDetailsPage.Values[4] + #13#10, False); + // UTF-8, no BOM. SaveStringToFile writes an AnsiString, but the installer + // reads this back with -Encoding UTF8 - so any non-ASCII character in the + // password came out mangled and the database rejected a password that was + // typed correctly, reported as "wrong username or password". + SetArrayLength(PwLines, 1); + PwLines[0] := DbCredsPage.Values[1]; + SaveStringsToUTF8FileWithoutBOM(PwFile, PwLines, False); end else PwFile := ''; @@ -1081,7 +1224,7 @@ begin ' -DbHost "' + DbDetailsPage.Values[0] + '"' + ' -DbPort ' + DbDetailsPage.Values[1] + ' -DbName "' + DbDetailsPage.Values[2] + '"' + - ' -DbUser "' + DbDetailsPage.Values[3] + '"'; + ' -DbUser "' + DbCredsPage.Values[0] + '"'; if PwFile <> '' then Args := Args + ' -DbPasswordFile "' + PwFile + '"'; end; @@ -1169,6 +1312,12 @@ end; ; Delegate to the same tested script rather than duplicating removal logic here. ; It removes the site, app pool, firewall rule and application directory, and ; deliberately does NOT drop the database or uninstall MySQL. -Filename: "powershell.exe"; \ +; +; Sysnative, for the same reason as the [Run] entry above: the uninstaller is a +; 32-bit process, so a bare "powershell.exe" is resolved through WOW64 to the +; 32-bit PowerShell, which cannot see the IIS provider at all. Removal then +; skipped the site, application pool and application - leaving them pointing at +; a directory it HAD deleted - while Windows reported a clean uninstall. +Filename: "{win}\Sysnative\WindowsPowerShell\v1.0\powershell.exe"; \ Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\shopdb-install.ps1"" -Stage uninstall -BundleRoot ""{app}"" -AppRoot ""{app}"" -OnFailure never"; \ RunOnceId: "ShopDBFlaskUninstall"; Flags: waituntilterminated runhidden diff --git a/deploy/windows/installer/build-installer.ps1 b/deploy/windows/installer/build-installer.ps1 index 080632fe..bba59a24 100644 --- a/deploy/windows/installer/build-installer.ps1 +++ b/deploy/windows/installer/build-installer.ps1 @@ -286,7 +286,7 @@ refresh-bundle-lock.ps1, read the diff, and commit the new lock). # --- summary ---------------------------------------------------------------- Write-Host '' Say " Bundle staged at: $BundleRoot" 'White' -foreach ($d in @('app', 'wheels', 'python', 'httpplatformhandler', 'urlrewrite', 'mysql')) { +foreach ($d in @('app', 'wheels', 'python', 'httpplatformhandler', 'urlrewrite', 'vcredist', 'mysqlclient', 'mysql')) { $p = Join-Path $BundleRoot $d if (Test-Path $p) { $mb = ((Get-ChildItem $p -Recurse -File | Measure-Object Length -Sum).Sum / 1MB) diff --git a/deploy/windows/installer/build-installer.sh b/deploy/windows/installer/build-installer.sh index 9062dc2e..3f527e5d 100755 --- a/deploy/windows/installer/build-installer.sh +++ b/deploy/windows/installer/build-installer.sh @@ -82,7 +82,7 @@ done echo "" echo "Bundle staged at: $BUNDLE" -for d in app wheels python httpplatformhandler urlrewrite mysql; do +for d in app wheels python httpplatformhandler urlrewrite vcredist mysqlclient mysql; do if [ -d "$BUNDLE/$d" ]; then printf ' %-20s %s\n' "$d" "$(du -sh "$BUNDLE/$d" | cut -f1)" else @@ -125,7 +125,8 @@ else echo " python/ python-3.14.x-amd64.exe" echo " httpplatformhandler/ httpPlatformHandler_amd64.msi" echo " urlrewrite/ rewrite_amd64.msi (client-IP rule; see README)" - echo " mysql/ mysql-8.0.x-winx64.msi (bundled-database option only)" + echo " vcredist/ VC_redist.x64.exe (MySQL requires it)" + echo " mysql/ mysql-8.4.x-winx64.msi (bundled-database option only)" echo "" echo " If the payload changed ON PURPOSE, regenerate and COMMIT the lock:" echo " pwsh ./refresh-bundle-lock.ps1 # review the diff" diff --git a/deploy/windows/installer/bundle-lock.json b/deploy/windows/installer/bundle-lock.json index d56fef7e..34bdb313 100644 --- a/deploy/windows/installer/bundle-lock.json +++ b/deploy/windows/installer/bundle-lock.json @@ -1,197 +1,240 @@ { "schema": 1, - "generated": "2026-08-03T15:45:11Z", + "generated": "2026-08-04T23:11:22Z", "pythontag": "cp314", "platform": "win_amd64", "note": "Exact third-party payload of the installer bundle. Regenerate with refresh-bundle-lock.ps1 and COMMIT the change as a reviewed dependency bump.", "payloads": { - "wheels": { - "required": true, - "files": { - "jinja2-3.1.6-py3-none-any.whl": { - "size": 134899, - "sha256": "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" - }, - "itsdangerous-2.2.0-py3-none-any.whl": { - "size": 16234, - "sha256": "c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef" - }, - "cffi-2.1.0-cp314-cp314-win_amd64.whl": { - "size": 187937, - "sha256": "1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb" - }, - "email_validator-2.3.0-py3-none-any.whl": { - "size": 35604, - "sha256": "80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4" - }, - "urllib3-2.7.0-py3-none-any.whl": { - "size": 131087, - "sha256": "9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" - }, - "flask_jwt_extended-4.7.3-py2.py3-none-any.whl": { - "size": 22698, - "sha256": "905ac807b52b5409bc9244dbcca434968c13ca6f9d91bffe7d4cb71e0e6231cb" - }, - "flask_sqlalchemy-3.1.1-py3-none-any.whl": { - "size": 25125, - "sha256": "4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0" - }, - "pycparser-3.0-py3-none-any.whl": { - "size": 48172, - "sha256": "b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" - }, - "requests-2.33.1-py3-none-any.whl": { - "size": 64947, - "sha256": "4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a" - }, - "blinker-1.9.0-py3-none-any.whl": { - "size": 8458, - "sha256": "ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc" - }, - "greenlet-3.5.0-cp314-cp314-win_amd64.whl": { - "size": 239835, - "sha256": "3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033" - }, - "marshmallow_sqlalchemy-1.5.0-py3-none-any.whl": { - "size": 16582, - "sha256": "3865232672f3dd38c4d5e4e85fdedce76904200742c3594948a2d11d0af93258" - }, - "alembic-1.18.4-py3-none-any.whl": { - "size": 263893, - "sha256": "a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a" - }, - "pyjwt-2.12.1-py3-none-any.whl": { - "size": 29726, - "sha256": "28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c" - }, - "mako-1.3.12-py3-none-any.whl": { - "size": 78521, - "sha256": "8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9" - }, - "waitress-3.0.2-py3-none-any.whl": { - "size": 56232, - "sha256": "c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e" - }, - "colorama-0.4.6-py2.py3-none-any.whl": { - "size": 25335, - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" - }, - "certifi-2026.4.22-py3-none-any.whl": { - "size": 135707, - "sha256": "3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a" - }, - "cachelib-0.13.0-py3-none-any.whl": { - "size": 20914, - "sha256": "8c8019e53b6302967d4e8329a504acf75e7bc46130291d30188a6e4e58162516" - }, - "pymysql-1.1.3-py3-none-any.whl": { - "size": 45356, - "sha256": "8164ba62c552f6105f3b11753352d0f16b90d1703ba67d81923d5a8a5d1c5289" - }, - "werkzeug-3.1.8-py3-none-any.whl": { - "size": 226459, - "sha256": "63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50" - }, - "idna-3.13-py3-none-any.whl": { - "size": 68629, - "sha256": "892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3" - }, - "python_dotenv-1.2.2-py3-none-any.whl": { - "size": 22101, - "sha256": "1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a" - }, - "flask_marshmallow-1.5.0-py3-none-any.whl": { - "size": 12161, - "sha256": "99951c77e5654111ed733811c6dc9310bfb4c3688c78a9e76f80b5ae0b2279a6" - }, - "markupsafe-3.0.3-cp314-cp314-win_amd64.whl": { - "size": 15341, - "sha256": "bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581" - }, - "flask_cors-6.0.2-py3-none-any.whl": { - "size": 13257, - "sha256": "e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a" - }, - "flask_caching-2.4.0-py3-none-any.whl": { - "size": 28727, - "sha256": "d15b8135f055c4f28f6f7dbcf8d36a3de4af1224def975ae6e0b43cbfa684486" - }, - "tzdata-2026.3-py2.py3-none-any.whl": { - "size": 348168, - "sha256": "dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931" - }, - "charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl": { - "size": 159634, - "sha256": "92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f" - }, - "sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl": { - "size": 2144204, - "sha256": "77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5" - }, - "marshmallow-4.3.0-py3-none-any.whl": { - "size": 49148, - "sha256": "46c4fe6984707e3cbd485dfebbf0a59874f58d695aad05c1668d15e8c6e13b46" - }, - "dnspython-2.8.0-py3-none-any.whl": { - "size": 331094, - "sha256": "01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af" - }, - "typing_extensions-4.15.0-py3-none-any.whl": { - "size": 44614, - "sha256": "f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548" - }, - "flask-3.1.3-py3-none-any.whl": { - "size": 103424, - "sha256": "f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c" - }, - "tabulate-0.10.0-py3-none-any.whl": { - "size": 39814, - "sha256": "f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3" - }, - "mysql_connector_python-9.7.0-cp314-cp314-win_amd64.whl": { - "size": 18197576, - "sha256": "5a5abbc152bc28cb2e64a04605ecd9941eff6b0dc5f9528cb84adb873e9a1e49" - }, - "Flask_Migrate-4.1.0-py3-none-any.whl": { - "size": 21237, - "sha256": "24d8051af161782e0743af1b04a152d007bad9772b2bca67b7ec1e8ceeb3910d" - }, - "cryptography-50.0.0-cp311-abi3-win_amd64.whl": { - "size": 3840395, - "sha256": "bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30" - }, - "click-8.3.3-py3-none-any.whl": { - "size": 110502, - "sha256": "a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613" - } - } - }, - "python": { - "required": true, - "files": { - "python-3.14.6-amd64.exe": { - "size": 30774112, - "sha256": "14b3e9a710a3fcf0bd9b55ab6b60412bd91227563f813fc49040cabc0209e0bd" - } - } - }, - "urlrewrite": { - "required": false, - "files": { - "rewrite_amd64_en-US.msi": { - "size": 6078464, - "sha256": "37342ff2f585f263f34f48e9de59eb1051d61015a8e967dbde4075716230a32a" - } - } - }, "httpplatformhandler": { - "required": true, "files": { "httpPlatformHandler_amd64.msi": { - "size": 557056, - "sha256": "90f8d4905a0ab4f2c95223b3c79e2807a0b74507747d240e43c4302e8db4b5ef" + "sha256": "90f8d4905a0ab4f2c95223b3c79e2807a0b74507747d240e43c4302e8db4b5ef", + "size": 557056 } - } + }, + "required": true + }, + "python": { + "files": { + "python-3.14.6-amd64.exe": { + "sha256": "14b3e9a710a3fcf0bd9b55ab6b60412bd91227563f813fc49040cabc0209e0bd", + "size": 30774112 + } + }, + "required": true + }, + "vcredist": { + "files": { + "VC_redist.x64.exe": { + "sha256": "cc0ff0eb1dc3f5188ae6300faef32bf5beeba4bdd6e8e445a9184072096b713b", + "size": 25635768 + } + }, + "required": false + }, + "mysql": { + "files": { + "mysql-8.4.6-winx64.msi": { + "sha256": "868885b2e221409f0170729bb30b2b2ce83440a1a4c735d19b3c225b51045762", + "size": 135081984 + } + }, + "required": false + }, + "wheels": { + "files": { + "email_validator-2.3.0-py3-none-any.whl": { + "sha256": "80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", + "size": 35604 + }, + "cachelib-0.13.0-py3-none-any.whl": { + "sha256": "8c8019e53b6302967d4e8329a504acf75e7bc46130291d30188a6e4e58162516", + "size": 20914 + }, + "flask_caching-2.4.0-py3-none-any.whl": { + "sha256": "d15b8135f055c4f28f6f7dbcf8d36a3de4af1224def975ae6e0b43cbfa684486", + "size": 28727 + }, + "typing_extensions-4.15.0-py3-none-any.whl": { + "sha256": "f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", + "size": 44614 + }, + "greenlet-3.5.0-cp314-cp314-win_amd64.whl": { + "sha256": "3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033", + "size": 239835 + }, + "charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl": { + "sha256": "92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", + "size": 159634 + }, + "colorama-0.4.6-py2.py3-none-any.whl": { + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "size": 25335 + }, + "Flask_Migrate-4.1.0-py3-none-any.whl": { + "sha256": "24d8051af161782e0743af1b04a152d007bad9772b2bca67b7ec1e8ceeb3910d", + "size": 21237 + }, + "markupsafe-3.0.3-cp314-cp314-win_amd64.whl": { + "sha256": "bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", + "size": 15341 + }, + "flask_marshmallow-1.5.0-py3-none-any.whl": { + "sha256": "99951c77e5654111ed733811c6dc9310bfb4c3688c78a9e76f80b5ae0b2279a6", + "size": 12161 + }, + "tzdata-2026.3-py2.py3-none-any.whl": { + "sha256": "dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", + "size": 348168 + }, + "python_dotenv-1.2.2-py3-none-any.whl": { + "sha256": "1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", + "size": 22101 + }, + "dnspython-2.8.0-py3-none-any.whl": { + "sha256": "01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", + "size": 331094 + }, + "pymysql-1.1.3-py3-none-any.whl": { + "sha256": "8164ba62c552f6105f3b11753352d0f16b90d1703ba67d81923d5a8a5d1c5289", + "size": 45356 + }, + "packaging-26.3-py3-none-any.whl": { + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", + "size": 129956 + }, + "flask_jwt_extended-4.7.3-py2.py3-none-any.whl": { + "sha256": "905ac807b52b5409bc9244dbcca434968c13ca6f9d91bffe7d4cb71e0e6231cb", + "size": 22698 + }, + "urllib3-2.7.0-py3-none-any.whl": { + "sha256": "9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", + "size": 131087 + }, + "marshmallow-4.3.0-py3-none-any.whl": { + "sha256": "46c4fe6984707e3cbd485dfebbf0a59874f58d695aad05c1668d15e8c6e13b46", + "size": 49148 + }, + "waitress-3.0.2-py3-none-any.whl": { + "sha256": "c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e", + "size": 56232 + }, + "idna-3.13-py3-none-any.whl": { + "sha256": "892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", + "size": 68629 + }, + "flask-3.1.3-py3-none-any.whl": { + "sha256": "f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", + "size": 103424 + }, + "werkzeug-3.1.8-py3-none-any.whl": { + "sha256": "63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", + "size": 226459 + }, + "flask_sqlalchemy-3.1.1-py3-none-any.whl": { + "sha256": "4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0", + "size": 25125 + }, + "marshmallow_sqlalchemy-1.5.0-py3-none-any.whl": { + "sha256": "3865232672f3dd38c4d5e4e85fdedce76904200742c3594948a2d11d0af93258", + "size": 16582 + }, + "alembic-1.18.4-py3-none-any.whl": { + "sha256": "a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", + "size": 263893 + }, + "tabulate-0.10.0-py3-none-any.whl": { + "sha256": "f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", + "size": 39814 + }, + "cffi-2.1.0-cp314-cp314-win_amd64.whl": { + "sha256": "1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", + "size": 187937 + }, + "jinja2-3.1.6-py3-none-any.whl": { + "sha256": "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", + "size": 134899 + }, + "cryptography-50.0.0-cp311-abi3-win_amd64.whl": { + "sha256": "bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", + "size": 3840395 + }, + "certifi-2026.4.22-py3-none-any.whl": { + "sha256": "3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", + "size": 135707 + }, + "sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl": { + "sha256": "77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", + "size": 2144204 + }, + "blinker-1.9.0-py3-none-any.whl": { + "sha256": "ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", + "size": 8458 + }, + "pycparser-3.0-py3-none-any.whl": { + "sha256": "b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", + "size": 48172 + }, + "flask_cors-6.0.2-py3-none-any.whl": { + "sha256": "e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a", + "size": 13257 + }, + "requests-2.33.1-py3-none-any.whl": { + "sha256": "4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", + "size": 64947 + }, + "pyjwt-2.12.1-py3-none-any.whl": { + "sha256": "28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", + "size": 29726 + }, + "click-8.3.3-py3-none-any.whl": { + "sha256": "a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", + "size": 110502 + }, + "mysql_connector_python-9.7.0-cp314-cp314-win_amd64.whl": { + "sha256": "5a5abbc152bc28cb2e64a04605ecd9941eff6b0dc5f9528cb84adb873e9a1e49", + "size": 18197576 + }, + "mako-1.3.12-py3-none-any.whl": { + "sha256": "8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", + "size": 78521 + }, + "itsdangerous-2.2.0-py3-none-any.whl": { + "sha256": "c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", + "size": 16234 + } + }, + "required": true + }, + "urlrewrite": { + "files": { + "rewrite_amd64_en-US.msi": { + "sha256": "37342ff2f585f263f34f48e9de59eb1051d61015a8e967dbde4075716230a32a", + "size": 6078464 + } + }, + "required": false + }, + "mysqlclient": { + "files": { + "libcrypto-3-x64.dll": { + "sha256": "e84ec61fc1b07a3c899ae39c1dc3f38591cdd310c043a388e7ab40a6828aa297", + "size": 5187216 + }, + "mysqldump.exe": { + "sha256": "329410b7a8ae3d68d38e6f5b781610770854d3f11b975060e231dae62c20dc46", + "size": 7142528 + }, + "mysql.exe": { + "sha256": "bd6880253c1853fc17b1e19129d6e5d9f9c86a2dccf2c7c9e6b878a10baa9770", + "size": 7177336 + }, + "libssl-3-x64.dll": { + "sha256": "e24fcd6c8ec31a14c97f520b5b960ed360c55ddf0935040eff297e511697633e", + "size": 788616 + } + }, + "required": false } } } diff --git a/deploy/windows/installer/bundle-lock.ps1 b/deploy/windows/installer/bundle-lock.ps1 index 7cb317ec..9ba92b30 100644 --- a/deploy/windows/installer/bundle-lock.ps1 +++ b/deploy/windows/installer/bundle-lock.ps1 @@ -50,6 +50,7 @@ $script:BundlePayloads = @( @{ Name = 'httpplatformhandler'; Required = $true; What = 'the IIS module that launches waitress' } @{ Name = 'urlrewrite'; Required = $false; What = 'IIS URL Rewrite, for the client-IP rule' } @{ Name = 'mysqlclient'; Required = $false; What = 'mysql/mysqldump, for backups against a remote database' } + @{ Name = 'vcredist'; Required = $false; What = 'the Visual C++ runtime MySQL requires' } @{ Name = 'mysql'; Required = $false; What = 'MySQL, for the bundled-database option' } ) @@ -75,14 +76,35 @@ function Get-FileDigest { } function Get-PayloadFiles { - # Every file in the directory, keyed by its path RELATIVE to that directory - # with forward slashes, so a lock generated on Windows reads the same from - # the Bash builder. + <# + Every file in the directory, keyed by its path RELATIVE to that directory + with forward slashes, so a lock generated on Windows reads the same from + the Bash builder. + + The root comes from Get-Item, NOT Resolve-Path, and the prefix is checked + before it is trimmed. Both matter, and a real install proved it: + + Inno extracts the bundle under C:\Users\ADMINI~1\AppData\Local\Temp\... - + an 8.3 SHORT path. Resolve-Path kept that short form while Get-ChildItem + returned the long one (Administrator), so the root was five characters + shorter than the prefix it was slicing off. Every relative path came out + mangled - 'wheels/heels/flask.whl' - and the verifier reported all 96 files + as simultaneously missing and unexpected. The payload was fine; the + comparison was not. + + Get-Item and Get-ChildItem go through the same provider, so their path + forms agree. The StartsWith guard means that if they ever disagree again + this fails loudly instead of inventing paths. + #> param([string] $Dir) $out = @{} if (-not (Test-Path $Dir)) { return $out } - $root = (Resolve-Path $Dir).Path.TrimEnd('\', '/') - foreach ($f in (Get-ChildItem $Dir -Recurse -File)) { + $rootItem = Get-Item -LiteralPath $Dir + $root = $rootItem.FullName.TrimEnd('\', '/') + foreach ($f in (Get-ChildItem -LiteralPath $rootItem.FullName -Recurse -File)) { + if (-not $f.FullName.StartsWith($root, [System.StringComparison]::OrdinalIgnoreCase)) { + throw ("cannot place '{0}' beneath '{1}' - path forms disagree (8.3 short name?)" -f $f.FullName, $root) + } $rel = $f.FullName.Substring($root.Length).TrimStart('\', '/').Replace('\', '/') $out[$rel] = @{ sha256 = (Get-FileDigest $f.FullName); size = $f.Length } } @@ -137,7 +159,9 @@ function Test-BundleLock { ) $problems = @() $payloads = Get-JsonProperty $Lock 'payloads' - if ($null -eq $payloads) { return @('bundle-lock.json has no "payloads" section') } + # ,@(...) for the same reason as the final return: a bare one-element array + # unrolls to a string, and the caller's .Count then measures the wrong thing. + if ($null -eq $payloads) { return ,@('bundle-lock.json has no "payloads" section') } foreach ($p in $script:BundlePayloads) { $name = $p.Name @@ -184,7 +208,11 @@ function Test-BundleLock { } } $problems += Test-WheelhouseCoversRequirements -BundleRoot $BundleRoot - return $problems + # The comma keeps this an ARRAY through the return. Without it PowerShell + # unrolls an empty result to $null and a single result to a bare string, and + # every caller that asks for .Count then behaves differently depending on how + # many problems there happen to be. + return ,$problems } function Test-WheelhouseCoversRequirements { diff --git a/deploy/windows/installer/shopdb-admin.ps1 b/deploy/windows/installer/shopdb-admin.ps1 index 4307aee3..43733b7a 100644 --- a/deploy/windows/installer/shopdb-admin.ps1 +++ b/deploy/windows/installer/shopdb-admin.ps1 @@ -195,6 +195,8 @@ function Get-DbParts { function Find-MysqlClient { $candidates = @( + 'C:\MySQL84\bin\mysql.exe', + 'C:\Program Files\MySQL\MySQL Server 8.4\bin\mysql.exe', 'C:\Program Files\MySQL\MySQL Server 8.0\bin\mysql.exe', 'C:\mysql56\bin\mysql.exe' ) + @(Get-ChildItem 'C:\Program Files\MySQL' -Filter mysql.exe -Recurse -EA SilentlyContinue | @@ -312,10 +314,38 @@ function Stop-App { function Restart-App { Head 'Restarting' - # Recycle rather than stop/start: it drains existing requests instead of - # cutting them off, and it is what a config change actually needs. - & $AppCmd recycle apppool /apppool.name:$AppPool 2>&1 | ForEach-Object { Say " $_" } - Start-Sleep -Seconds 2 + # A recycle CANNOT start something that is stopped - it is a no-op on a + # stopped pool. Restarting after a failed upgrade, which is exactly when the + # pool is stopped, therefore did nothing and then reported "did not respond" + # in red, as though the application were broken. Start it first when needed. + Import-Module WebAdministration -ErrorAction SilentlyContinue + $started = $true + try { + if (Test-Path "IIS:\AppPools\$AppPool") { + $started = ((Get-Item "IIS:\AppPools\$AppPool").State -eq 'Started') + } + } catch { } + if (-not $started) { + Say ' the application pool is stopped; starting it' 'Yellow' + Start-WebAppPool -Name $AppPool -ErrorAction SilentlyContinue + Start-Sleep -Seconds 3 + } else { + # Recycle rather than stop/start: it drains existing requests instead of + # cutting them off, and it is what a config change actually needs. + & $AppCmd recycle apppool /apppool.name:$AppPool 2>&1 | ForEach-Object { Say " $_" } + if ($LASTEXITCODE -ne 0) { Say " recycle reported exit $LASTEXITCODE" 'Yellow' } + Start-Sleep -Seconds 2 + } + + # A stopped SITE answers nothing however healthy the pool is. + try { + $siteNow = Get-Website -Name $SiteName -ErrorAction SilentlyContinue + if ($siteNow -and $siteNow.State -ne 'Started') { + Say ' the site is stopped; starting it' 'Yellow' + Start-Website -Name $SiteName -ErrorAction SilentlyContinue + Start-Sleep -Seconds 2 + } + } catch { } try { $r = Invoke-WebRequest -Uri ((Get-Deployment).BaseUrl + '/') -UseBasicParsing -TimeoutSec 30 Say (" back up (HTTP {0})" -f $r.StatusCode) 'Green' @@ -370,9 +400,28 @@ function Backup-Db { Head 'Database backup' $db = Get-DbParts if (-not $db) { Say ' no .env found - cannot determine the database' 'Red'; return } + $usingDefault = -not $Dest if (-not $Dest) { $Dest = 'C:\ProgramData\ShopDB-Flask\backups' } if (-not (Test-Path $Dest)) { New-Item -ItemType Directory -Path $Dest -Force | Out-Null } + # A dump holds every row, including the users table and its password hashes. + # A directory created under ProgramData INHERITS Users:RX, so those hashes + # were readable by every authenticated user on the server whenever this + # command created the directory rather than the installer. + # + # Re-applied on every backup, not only on creation, because this may be + # repairing a directory made by an earlier version. + # + # Only for the default location. A path the operator named is theirs, and + # silently rewriting its ACL is not this command's business - say so instead. + if ($usingDefault) { + & icacls.exe $Dest '/inheritance:r' ` + '/grant' 'BUILTIN\Administrators:(OI)(CI)(F)' ` + '/grant' 'NT AUTHORITY\SYSTEM:(OI)(CI)(F)' 2>&1 | Out-Null + } else { + Say ' note: this dump contains password hashes - check who can read that directory' 'Yellow' + } + $mysql = Find-MysqlClient if (-not $mysql) { Say ' mysql client not found' 'Red'; return } $dump = Join-Path (Split-Path $mysql -Parent) 'mysqldump.exe' diff --git a/deploy/windows/installer/shopdb-install.ps1 b/deploy/windows/installer/shopdb-install.ps1 index feae3a81..d7c04854 100644 --- a/deploy/windows/installer/shopdb-install.ps1 +++ b/deploy/windows/installer/shopdb-install.ps1 @@ -49,7 +49,7 @@ [CmdletBinding()] param( - # Stage 0 installs the bundled MySQL 8.0 (greenfield sites only). It is NOT in + # Stage 0 installs the bundled MySQL 8.4 LTS (greenfield sites only). It is NOT in # 'all': a site with an existing MySQL must never have a second server dropped # on top of it, and that is the common case (docs/DEPLOY-WINDOWS-IIS.md: the # target box already runs the classic ASP shopdb against MySQL 5.6). Run it @@ -57,10 +57,16 @@ param( [ValidateSet('0','1','2','3','4','5','all','uninstall')] [string] $Stage = 'all', # Bundled-MySQL (stage 0) settings. The root and app passwords are GENERATED, # never supplied: see New-Secret. Root is shown once and not persisted. - [string] $MysqlRoot = 'C:\Program Files\MySQL\MySQL Server 8.0', + # SPACE-FREE, deliberately - same reason as the Python target below. msiexec + # takes public properties as PROPERTY=value on the command line, and a value + # containing spaces has to be quoted as INSTALLDIR="C:\..." - quoting the + # whole PROPERTY=value token instead makes msiexec reject the command line + # and pop its usage dialog, which is what 'the bundled MySQL database could + # not be installed' looked like from the outside. + [string] $MysqlRoot = 'C:\MySQL84', [string] $MysqlDataDir = 'C:\ProgramData\MySQL\data', [string] $MysqlIni = 'C:\ProgramData\MySQL\my.ini', - [string] $MysqlService = 'MySQL80', + [string] $MysqlService = 'MySQL84', [Parameter(Mandatory=$true)] [string] $BundleRoot, [string] $AppRoot = 'C:\shopdb-flask', [int] $SitePort = 8090, @@ -149,6 +155,8 @@ try { if (-not (Test-Path $script:LogDir)) { New-Item -ItemType Directory -Path catch { $script:LogDir = $env:TEMP } $script:LogPath = Join-Path $script:LogDir ("shopdb-install-{0}.log" -f (Get-Date -Format 'yyyyMMdd-HHmmss')) $script:Created = New-Object System.Collections.ArrayList # for rollback +$script:RebootPending = $false +$script:MysqlAlreadyOurs = $false # Was this server already stamped by a previous run of THIS installer, as observed # BEFORE stage 2 writes its own stamp? Stage 4 needs the answer to know whether the # IIS objects it is about to reconcile are its own. $null means nothing has looked @@ -175,7 +183,146 @@ function Fail { throw $Message } -function Track { param([string] $Kind, [string] $Id) $null = $script:Created.Add(@{Kind=$Kind; Id=$Id}) } +# ----------------------------------------------------------------------------- +# DURABLE INSTALL RECORD +# +# "Is this a re-run of MY install?" used to be answered from whatever the machine +# happened to look like: a MySQL service exists, the database has tables, the +# site exists, the venv exists. None of those record WHO created them, so a retry +# after a failed first attempt was repeatedly mistaken for an upgrade of somebody +# else's working system. That single confusion produced several separate dead +# ends, the worst being a failed first install demanding a mandatory backup of +# its own half-written database and then refusing to prune tables it had created +# itself minutes earlier. +# +# This file answers the question instead. It is written when provisioning +# STARTS, not when it finishes, so a run that dies halfway still leaves a record +# the next run can read - which is the whole point, because the run that dies +# halfway is the one whose retry needs to know. +# ----------------------------------------------------------------------------- +$script:StateFile = Join-Path $env:ProgramData 'ShopDB-Flask\install-state.json' +$script:InstallState = $null + +function Get-StateValue { + # Set-StrictMode makes reading a missing property a terminating error, so + # every read of a file that may be older, hand-edited or truncated goes + # through here rather than dotting into it directly. + param($Object, [string] $Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + +function Save-InstallState { + param($State) + # Never fatal. The record is an improvement on guessing; failing to write it + # only returns the installer to the behaviour it had before, so a permissions + # problem here must not stop an install that is otherwise fine. + try { + $dir = Split-Path $script:StateFile -Parent + if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } + ($State | ConvertTo-Json -Depth 6) | Set-Content -Path $script:StateFile -Encoding UTF8 + Protect-File $script:StateFile + } catch { + Write-Log "could not write the install record: $($_.Exception.Message)" 'WARN' + } +} + +function Get-InstallState { + if (-not (Test-Path $script:StateFile)) { return $null } + try { + $raw = Get-Content $script:StateFile -Raw -ErrorAction Stop + if (-not $raw) { return $null } + $obj = $raw | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Log 'the install record is unreadable; falling back to inspecting the server' 'WARN' + return $null + } + # A record describing a DIFFERENT directory says nothing about this one. + $recorded = Get-StateValue $obj 'approot' + if ($recorded -and -not (Test-SamePath $recorded $AppRoot)) { return $null } + return $obj +} + +function Initialize-InstallState { + $state = Get-InstallState + if ($null -eq $state) { + # No record at all. Either this is a genuinely new server, or an install + # that predates this file. Distinguish by the version stamp, and when in + # doubt choose the SAFE side: an established install, which keeps the + # mandatory pre-upgrade backup. Guessing "first run" for a server holding + # real data would let prune --force loose on it. + $stamped = Test-Path (Join-Path $AppRoot '.installed-version') + $state = [PSCustomObject]@{ + schema = 1 + approot = $AppRoot + firstrunstarted = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + firstruncompleted = $(if ($stamped) { 'predates-this-record' } else { $null }) + created = [PSCustomObject]@{} + } + if ($stamped) { + Write-Log 'no install record found, but this server carries a version stamp' 'WARN' + Write-Log ' treating it as an established install, so the pre-upgrade backup stays mandatory' 'WARN' + } + Save-InstallState $state + } + $script:InstallState = $state +} + +function Test-FirstProvisioning { + # TRUE only while OUR first provisioning of this directory is unfinished - + # including every retry of it. Whatever is in the database at that point was + # put there by an earlier attempt of this same install, not by a running site. + if ($null -eq $script:InstallState) { return $false } + return ($null -eq (Get-StateValue $script:InstallState 'firstruncompleted')) +} + +function Complete-InstallState { + if ($null -eq $script:InstallState) { return } + if ($null -ne (Get-StateValue $script:InstallState 'firstruncompleted')) { return } + $script:InstallState.firstruncompleted = + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + Save-InstallState $script:InstallState +} + +function Track { + param([string] $Kind, [string] $Id) + $null = $script:Created.Add(@{Kind=$Kind; Id=$Id}) + # Persist as well. $script:Created lives only in memory, so a run that died + # left nothing behind saying what it had already made, and the next run was + # back to inferring it from the machine. + try { + if ($null -ne $script:InstallState) { + $created = Get-StateValue $script:InstallState 'created' + if ($null -eq $created) { + $created = [PSCustomObject]@{} + $script:InstallState | Add-Member -NotePropertyName 'created' -NotePropertyValue $created -Force + } + $list = @(Get-StateValue $created $Kind | Where-Object { $_ }) + if ($list -notcontains $Id) { + $created | Add-Member -NotePropertyName $Kind -NotePropertyValue (@($list) + $Id) -Force + Save-InstallState $script:InstallState + } + } + } catch { } +} + +function Get-CreatedItems { + # What THIS installer recorded creating, by kind. Empty when there is no + # record, which callers must treat as "unknown", never as "nothing". + # + # The leading comma is load-bearing. Returning a zero-length array from a + # PowerShell function UNROLLS it to $null, so the caller gets $null and + # $null.Count is a terminating error under Set-StrictMode - the exact fault + # that made bundle verification fail on every install earlier. Comma-wrapping + # emits the array itself as a single object, so an empty result stays an + # empty array. + param([string] $Kind) + if ($null -eq $script:InstallState) { return ,@() } + $created = Get-StateValue $script:InstallState 'created' + return ,@(Get-StateValue $created $Kind | Where-Object { $_ }) +} function Invoke-Native { # Runs an external command and fails loudly. @@ -188,50 +335,81 @@ function Invoke-Native { # $ErrorActionPreference='Stop' turns that into a terminating error. # 3. Argument VALUES must never be logged: a DATABASE_URL or password would # land in the log we invite operators to send to support. + # 4. Some installers succeed with a NON-ZERO code. 3010 and 1641 both mean + # "done, reboot required" - the VC++ redistributable returns 3010 on a + # server with a pending file rename, which is an ordinary state on a + # freshly patched box. Treating it as failure aborts a correct install. param([string] $Exe, [string[]] $Arguments, [string] $What, [string] $StdinFile = '', - [int] $TimeoutSec = 0) + [int] $TimeoutSec = 0, [int[]] $OkExit = @(0)) Write-Log (" exec {0} ({1} args)" -f (Split-Path $Exe -Leaf), $Arguments.Count) # Start-Process -ArgumentList joins the array with spaces WITHOUT quoting, so # any argument containing whitespace gets split into several. That is how # "IIS AppPool\name:(OI)(CI)RX" became the parameter "IIS" (icacls exit 87), # and how TargetDir=C:\Program Files\... installed Python into C:\Program\. $quoted = $Arguments | ForEach-Object { - if ($_ -match '\s' -and $_ -notmatch '^".*"$') { '"' + $_ + '"' } else { $_ } + if ($_ -notmatch '\s' -or $_ -match '^".*"$') { $_ } + # PROPERTY=value (msiexec, msbuild): the VALUE is quoted, never the whole + # token. "INSTALLDIR=C:\Program Files\..." is rejected outright, whereas + # INSTALLDIR="C:\Program Files\..." is what the tool expects. + elseif ($_ -match '^([A-Za-z_][A-Za-z0-9_]*)=(.*)$') { '{0}="{1}"' -f $Matches[1], $Matches[2] } + else { '"' + $_ + '"' } } $so = [System.IO.Path]::GetTempFileName() $se = [System.IO.Path]::GetTempFileName() try { + # NO -Wait when a timeout is wanted. -Wait blocks inside Start-Process + # until the child exits, so by the time the timeout check below ran the + # process had ALWAYS exited and the whole block was dead code - every + # -TimeoutSec on every MSI call was doing nothing. Start without -Wait + # and do the waiting here, where it can be bounded. + $common = @{ FilePath = $Exe; ArgumentList = $quoted; PassThru = $true + NoNewWindow = $true + RedirectStandardOutput = $so; RedirectStandardError = $se } if ($StdinFile) { # Feeding SQL as a file on stdin, not as -e "source ": the mysql # client reads that path as a DATABASE NAME and fails with # ERROR 1049 Unknown database ''. - $p = Start-Process -FilePath $Exe -ArgumentList $quoted -Wait -PassThru ` - -NoNewWindow -RedirectStandardInput $StdinFile ` - -RedirectStandardOutput $so -RedirectStandardError $se - } else { - $p = Start-Process -FilePath $Exe -ArgumentList $quoted -Wait -PassThru ` - -NoNewWindow -RedirectStandardOutput $so -RedirectStandardError $se - } - # A hung child must not hang the install. msiexec in particular deadlocks - # when a second Windows Installer transaction holds the global mutex: two - # msiexec processes sat there for 40 minutes with no output and no error, - # and -Wait alone gives you no way out. - if ($TimeoutSec -gt 0 -and -not $p.HasExited) { - if (-not $p.WaitForExit($TimeoutSec * 1000)) { - try { $p.Kill() } catch { } - Fail "$What timed out after $TimeoutSec seconds" @' -The command was killed. For an MSI this usually means another Windows Installer -transaction held the global mutex: check for stray msiexec processes -(Get-Process msiexec), end them, then re-run this stage. -'@ - } + $common['RedirectStandardInput'] = $StdinFile } + # -Wait ALWAYS. Dropping it to make -TimeoutSec enforceable left + # $p.ExitCode unreadable: a Python install that had actually worked + # reported "Python install failed (exit )" with an empty code, because + # PowerShell only reliably populates ExitCode on a waited process. + # + # So the trade is deliberate. Exit codes are load-bearing here - 1639 vs + # 1603 vs 3010 is the whole diagnosis - and a bounded wait is not worth + # losing them for. -TimeoutSec is therefore ADVISORY: it is recorded in + # the log so a hang is identifiable, but the wait is unbounded. + $common['Wait'] = $true + if ($TimeoutSec -gt 0) { Write-Log (" (expected to finish within {0}s)" -f $TimeoutSec) } + $p = Start-Process @common + + # NOTE: msiexec can deadlock when another Windows Installer transaction + # holds the global mutex - two msiexec processes once sat for 40 minutes + # with no output and no error. That is why the expected duration is + # logged above: an operator seeing no progress well past it should check + # for stray msiexec processes (Get-Process msiexec) and end them. $out = @() if (Test-Path $so) { $out += Get-Content $so -ErrorAction SilentlyContinue } if (Test-Path $se) { $out += Get-Content $se -ErrorAction SilentlyContinue } - if ($p.ExitCode -ne 0) { + $code = $null + try { $code = $p.ExitCode } catch { } + if ($null -eq $code) { + # Never silently treat "cannot tell" as success. $out | Select-Object -Last 8 | ForEach-Object { Write-Log " $_" 'FAIL' } - Fail "$What failed (exit $($p.ExitCode))" + Fail "$What finished but its exit code could not be read" ` + 'Check the log above and the tool''s own log before re-running.' + } + if ($OkExit -notcontains $code) { + $out | Select-Object -Last 8 | ForEach-Object { Write-Log " $_" 'FAIL' } + Fail "$What failed (exit $code)" + } + if ($code -ne 0) { + # Accepted, but say so: 3010/1641 mean the server needs a reboot to + # finish, and an operator who never sees that wonders later why + # something only half works. + Write-Log ("$What returned {0} - completed, but this server needs a reboot to finish" -f $code) 'WARN' + $script:RebootPending = $true } return $out } @@ -240,9 +418,18 @@ transaction held the global mutex: check for stray msiexec processes function Protect-File { # Owner-only ACL: Administrators + SYSTEM, inheritance broken. - param([string] $Path) - Invoke-Native 'icacls.exe' @($Path,'/inheritance:r', - '/grant','BUILTIN\Administrators:(F)','/grant','NT AUTHORITY\SYSTEM:(F)') 'ACL' + # + # -Directory additionally makes the two grants INHERITABLE. Without (OI)(CI) + # the grants apply to the directory alone, so files written into it later - + # backups taken by the console, not by this script - are not covered. + param([string] $Path, [switch] $Directory) + $admins = 'BUILTIN\Administrators:(F)' + $system = 'NT AUTHORITY\SYSTEM:(F)' + if ($Directory) { + $admins = 'BUILTIN\Administrators:(OI)(CI)(F)' + $system = 'NT AUTHORITY\SYSTEM:(OI)(CI)(F)' + } + Invoke-Native 'icacls.exe' @($Path,'/inheritance:r','/grant',$admins,'/grant',$system) 'ACL' } function New-Secret { @@ -253,6 +440,22 @@ function New-Secret { return [Convert]::ToBase64String($b).Replace('+','-').Replace('/','_').TrimEnd('=') } +function Resolve-IisPath { + # IIS stores physicalPath as it was typed, so it can carry %SystemDrive%-style + # variables and a trailing backslash. A raw string compare therefore misses + # paths that ARE ours - on a hand-built deployment that means stage 4 fails to + # recognise its own application, and the uninstaller fails to recognise what + # it may remove. + param([string] $Path) + if (-not $Path) { return '' } + return [System.Environment]::ExpandEnvironmentVariables($Path).TrimEnd('\').ToLowerInvariant() +} + +function Test-SamePath { + param([string] $A, [string] $B) + return (Resolve-IisPath $A) -eq (Resolve-IisPath $B) +} + # Paths derived once. $Py = Join-Path $AppRoot 'venv\Scripts\python.exe' $Flask = Join-Path $AppRoot 'venv\Scripts\flask.exe' @@ -261,6 +464,19 @@ $EnvFile = Join-Path $AppRoot '.env' $WheelDir = Join-Path $BundleRoot 'wheels' $AppSource = Join-Path $BundleRoot 'app' +# Dot-sourced HERE, at script scope, not inside Assert-BundleIntegrity. Dot- +# sourcing inside a function loads into that FUNCTION's scope, so every helper +# it defines - Get-JsonProperty, Read-BundleLock, Test-BundleLock - disappears +# the moment the function returns. Assert-BundleIntegrity itself worked; the +# next caller, Get-WheelhousePythonTag, died with "The term 'Get-JsonProperty' +# is not recognized" on any run where a venv already existed, which is every +# retry after a part-completed install. +# +# Absent on a stage that runs without the bundle (uninstall), so this is a +# guarded load and Assert-BundleIntegrity still fails loudly if it is missing. +$BundleLockLib = Join-Path $BundleRoot 'bundle-lock.ps1' +if (Test-Path $BundleLockLib) { . $BundleLockLib } + # ============================================================================= # Version awareness and database safety (upgrades) @@ -296,8 +512,12 @@ function Assert-BundleIntegrity { Fail 'bundle-lock.json is missing from the bundle' ` 'This bundle was not assembled by build-installer.ps1/.sh. Rebuild it.' } - . $checker $lock = Read-BundleLock $lockFile + # NO @() here. Test-BundleLock returns ,$problems, which already hands the + # array back intact - wrapping it again nests it, and .Count becomes 1 no + # matter how many problems there are. That turns a byte-perfect bundle into + # "1 problem(s)" on every install. Applying the same guard at both ends was + # worse than applying it at neither. $problems = Test-BundleLock -BundleRoot $BundleRoot -Lock $lock if ($problems.Count -gt 0) { foreach ($p in $problems) { Write-Log " $p" 'FAIL' } @@ -409,7 +629,10 @@ function Find-MysqlTool { ) foreach ($b in $bundled) { if (Test-Path $b) { return $b } } - $roots = @('C:\Program Files\MySQL', 'C:\mysql56\bin', 'C:\Program Files (x86)\MySQL') + # C:\MySQL84 first: that is where the bundled server installs (space-free, + # see $MysqlRoot). The Program Files paths cover a server MySQL put there + # by hand or by an older build. + $roots = @('C:\MySQL84', 'C:\Program Files\MySQL', 'C:\mysql56\bin', 'C:\Program Files (x86)\MySQL') foreach ($r in $roots) { if (Test-Path $r) { $hit = Get-ChildItem $r -Filter $Name -Recurse -ErrorAction SilentlyContinue | @@ -487,8 +710,13 @@ function Backup-Database { $dir = Join-Path $env:ProgramData 'ShopDB-Flask\backups' if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null - Protect-File $dir } + # RE-APPLIED EVERY TIME, not only on creation. The console creates this + # directory too (shopdb-admin.ps1 backup), and one created there inherits + # ProgramData's Users:RX - so dumps containing the users table and its + # password hashes became readable by every authenticated user on the server. + # The old create-only guard meant the installer could never repair that. + Protect-File $dir -Directory $file = Join-Path $dir ("{0}-{1}-{2}.sql" -f $db.Name, $Reason, (Get-Date -Format 'yyyyMMdd-HHmmss')) Write-Log "backing up $($db.Name) before migrating" @@ -608,19 +836,19 @@ except Exception: } # ============================================================================= -# STAGE 0 - bundled MySQL 8.0 (greenfield sites only) +# STAGE 0 - bundled MySQL 8.4 LTS (greenfield sites only) # ============================================================================= # The MSI installs BINARIES ONLY: no service, no data directory, no config. The # full sequence is msiexec -> write my.ini -> --initialize-insecure -> --install # -> Start-Service -> bootstrap SQL. Every step below has bitten us once. function Invoke-Stage0 { - Write-Log 'STAGE 0: bundled MySQL 8.0' 'STEP' + Write-Log 'STAGE 0: bundled MySQL 8.4 LTS' 'STEP' $msi = Get-ChildItem (Join-Path $BundleRoot 'mysql') -Filter '*.msi' -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $msi) { Fail "no MySQL MSI in $BundleRoot\mysql" @' -Stage 0 installs the BUNDLED database. Add mysql\mysql-8.0.x-winx64.msi to the +Stage 0 installs the BUNDLED database. Add mysql\mysql-8.4.x-winx64.msi to the bundle, or skip stage 0 and use the site's existing server (the common case). '@ } @@ -628,14 +856,33 @@ bundle, or skip stage 0 and use the site's existing server (the common case). # Refuse to stack a second server on an existing one. Two servers fighting # over 3306, or a silent takeover of the classic ASP app's database, is far # worse than stopping here. - $existing = Get-Service -Name 'MySQL*' -ErrorAction SilentlyContinue - if ($existing) { - Fail ("MySQL service already present: {0}" -f (($existing | ForEach-Object Name) -join ', ')) @' + # OUR service is not "someone else's MySQL". This refused to run when the + # only service present was the MySQL84 that a previous stage 0 registered - + # so every retry of a part-completed bundled install dead-ended, while the + # wizard told the operator that re-running was safe. + $existing = @(Get-Service -Name 'MySQL*' -ErrorAction SilentlyContinue) + $foreign = @($existing | Where-Object { $_.Name -ne $MysqlService }) + if ($foreign.Count -gt 0) { + Fail ("MySQL service already present: {0}" -f (($foreign | ForEach-Object Name) -join ', ')) @' This box already runs MySQL. Do NOT install the bundled server on top of it: it would collide on port 3306 and could disrupt the existing application. Skip stage 0 and point the installer at the existing server with -DbHost. '@ } + $ours = @($existing | Where-Object { $_.Name -eq $MysqlService }) + if ($ours.Count -gt 0) { + # Ours, from an earlier run. Bring it up if it is down and let the rest of + # the stage run: the bootstrap SQL is idempotent (CREATE ... IF NOT EXISTS + # plus ALTER USER), so it re-asserts the password and the grants and + # leaves .dbpass agreeing with the server. + Write-Log "$MysqlService is already registered by a previous run" 'OK' + if ($ours[0].Status -ne 'Running') { + Write-Log "starting $MysqlService" + Start-Service -Name $MysqlService -ErrorAction SilentlyContinue + Start-Sleep -Seconds 3 + } + $script:MysqlAlreadyOurs = $true + } $inUse = Get-NetTCPConnection -LocalPort $DbPort -State Listen -ErrorAction SilentlyContinue if ($inUse) { Fail "port $DbPort is already in use" 'Free the port, or use the existing server.' } @@ -663,10 +910,43 @@ Skip stage 0 and point the installer at the existing server with -DbHost. Write-Log "could not start msiserver: $($_.Exception.Message)" 'WARN' } } + # The Visual C++ runtime, BEFORE MySQL. MySQL's MSI carries a launch + # condition requiring it, and a bare Windows Server does not ship it - + # so on a fresh box the MSI aborts in about a second with the generic + # exit 1603 and no indication of what was missing. That is what a real + # Server 2019 install did. + $vcMarker = Join-Path $env:windir 'system32\vcruntime140.dll' + if (Test-Path $vcMarker) { + Write-Log 'Visual C++ runtime already present' 'OK' + } else { + $vc = Get-ChildItem (Join-Path $BundleRoot 'vcredist') -Filter '*.exe' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if (-not $vc) { + Fail 'the Visual C++ runtime is missing and no redistributable is in the bundle' @' +MySQL will not install without it - its MSI fails with a bare exit 1603. +Add vcredist\VC_redist.x64.exe to the bundle and rebuild, or install the +Microsoft Visual C++ 2015-2022 Redistributable (x64) on this server by hand. +'@ + } + Write-Log "installing $($vc.Name) (MySQL requires it)" + if (-not $WhatIfOnly) { + Invoke-Native $vc.FullName @('/install','/quiet','/norestart') ` + 'Visual C++ runtime' -TimeoutSec 600 -OkExit 0,3010,1641 + Write-Log 'Visual C++ runtime installed' 'OK' + } + } + # Quiet MSI. INSTALLDIR must have no trailing backslash or the MSI mis-parses it. Write-Log "installing $($msi.Name) (125MB, takes a minute)" + # /l*v: without the MSI's own log, exit 1603 says only "something went + # wrong" - it names neither the action nor the reason, and it is the most + # common MySQL failure code. Costs nothing and turns the next failure + # into something readable. + $msiLog = Join-Path $script:LogDir ('mysql-msi-{0}.log' -f (Get-Date -Format 'yyyyMMdd-HHmmss')) + Write-Log "MSI log: $msiLog" Invoke-Native 'msiexec.exe' @('/i', $msi.FullName, '/quiet', '/norestart', - "INSTALLDIR=$MysqlRoot") 'MySQL MSI' -TimeoutSec 900 + '/l*v', $msiLog, + "INSTALLDIR=$MysqlRoot") 'MySQL MSI' -TimeoutSec 900 -OkExit 0,3010 } # An installer exit code of 0 does NOT mean it did what you asked - the Python # bootstrapper returned 0 while installing to the wrong directory. Verify. @@ -695,6 +975,14 @@ log-error=$MysqlDataDir\mysql-error.log Write-Log "wrote $MysqlIni" Track 'mysql-files' $MysqlIni + # Everything from here to the bootstrap creates the server. Skip it wholesale + # when the service is already ours: the datadir check would refuse a + # populated directory, --initialize-insecure would refuse it too, and + # --install would fail on an existing service name. + if ($script:MysqlAlreadyOurs) { + Write-Log 'server already exists from a previous run; skipping create' 'OK' + } else { + # --- data directory ----------------------------------------------------- # --initialize-insecure REFUSES a non-empty datadir, so a previous failed # attempt must be cleared before retrying. @@ -724,9 +1012,32 @@ data, STOP: you are about to destroy a database. Set-Service -Name $MysqlService -StartupType Automatic Write-Log "service $MysqlService running" 'OK' + } # end of the create-the-server block + # --- bootstrap ---------------------------------------------------------- # --initialize-insecure leaves root with an EMPTY password, so this must run # immediately. Generated, never supplied; root is shown once and not stored. + # + # On a server this stage already built, root HAS a password, so the + # --skip-password connection below cannot authenticate. The database, the + # user and the handoff all exist from that run, so there is nothing to do - + # skip rather than fail. Without the handoff there is no safe automatic + # recovery, so say what to do instead of guessing. + if ($script:MysqlAlreadyOurs) { + if (Test-Path (Join-Path $AppRoot '.dbpass')) { + Write-Log 'database, user and password handoff already exist; skipping bootstrap' 'OK' + Write-Log 'stage 0 complete' 'OK' + return + } + Fail 'MySQL is installed by a previous run but its password handoff is gone' @' +The bundled server exists and root already has a generated password, so this +stage cannot re-create the application user, and .dbpass is not there to reuse. + +Either point the installer at this server with -DbHost 127.0.0.1 and the +password from C:\shopdb-flask\.env, or remove the MySQL84 service and its data +directory to install cleanly. +'@ + } $rootPass = New-Secret -Bytes 24 $appPass = New-Secret -Bytes 24 # Written OUTSIDE %TEMP% and ACL'd BEFORE the secrets go in - a file created @@ -739,8 +1050,23 @@ data, STOP: you are about to destroy a database. CREATE DATABASE IF NOT EXISTS $DbName CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE USER IF NOT EXISTS '$DbUser'@'localhost' IDENTIFIED BY '$appPass'; CREATE USER IF NOT EXISTS '$DbUser'@'127.0.0.1' IDENTIFIED BY '$appPass'; +-- ::1 as well. On current Windows 'localhost' resolves to the IPv6 loopback +-- FIRST, so an operator who types localhost instead of 127.0.0.1 arrives as +-- '$DbUser'@'::1' - an account that did not exist, and MySQL answers "Access +-- denied" naming a host the operator never typed. +CREATE USER IF NOT EXISTS '$DbUser'@'::1' IDENTIFIED BY '$appPass'; +-- ALTER after CREATE, because CREATE USER IF NOT EXISTS is a NO-OP on an +-- existing user and does NOT change its password. This stage generates a fresh +-- password every run and overwrites .dbpass with it, so on any re-run after a +-- partial failure the file held a password the server had never been told - +-- "Access denied" while the operator was looking straight at the right file. +-- ALTER makes the stored password and the handoff agree, every time. +ALTER USER '$DbUser'@'localhost' IDENTIFIED BY '$appPass'; +ALTER USER '$DbUser'@'127.0.0.1' IDENTIFIED BY '$appPass'; +ALTER USER '$DbUser'@'::1' IDENTIFIED BY '$appPass'; GRANT ALL PRIVILEGES ON $DbName.* TO '$DbUser'@'localhost'; GRANT ALL PRIVILEGES ON $DbName.* TO '$DbUser'@'127.0.0.1'; +GRANT ALL PRIVILEGES ON $DbName.* TO '$DbUser'@'::1'; ALTER USER 'root'@'localhost' IDENTIFIED BY '$rootPass'; FLUSH PRIVILEGES; "@ @@ -789,7 +1115,6 @@ FLUSH PRIVILEGES; $rootPass ) -Encoding UTF8 Write-Log "MySQL root password written to $rootFile (Administrators and SYSTEM only)" 'OK' - Write-Log 'MYSQLROOTFILE:' + $rootFile Write-Log 'stage 0 complete' 'OK' } @@ -828,10 +1153,12 @@ function Invoke-Stage2 { } else { Write-Log "installing Python (all users) from $($pyInstaller.Name)" if (-not $WhatIfOnly) { + # /norestart matters: without it the bootstrapper is free to reboot + # the server on its own, mid-install, with the wizard still running. Invoke-Native $pyInstaller.FullName @( - '/quiet','InstallAllUsers=1','PrependPath=0','Include_launcher=1', + '/quiet','/norestart','InstallAllUsers=1','PrependPath=0','Include_launcher=1', 'Include_test=0','AssociateFiles=0',"TargetDir=$pyTarget" - ) 'Python install' + ) 'Python install' -TimeoutSec 900 -OkExit 0,3010,1641 Track 'python' $pyTarget } } @@ -911,16 +1238,42 @@ what is on this server, or restore a backup taken before the upgrade. # Uninstall FIRST, while the code is still importable - the # registry entry outlives the directory otherwise, and the app # then fails to load a plugin it still believes is installed. + $deregistered = $false try { if (Test-Path $Flask) { Push-Location $AppRoot - $env:FLASK_APP = 'shopdb' - & $Flask plugin uninstall $dir.Name 2>&1 | - ForEach-Object { Write-Log " $_" } - Pop-Location + try { + $env:FLASK_APP = 'shopdb' + # --yes is REQUIRED: the command carries a click + # confirmation_option, which aborts with exit 1 + # when there is no console to answer the prompt. + # Without it this deregistration could never once + # have succeeded. + # + # EAP bracket for the reason documented on + # Invoke-Native: '2>&1' under EAP 'Stop' turns any + # stderr line into a terminating error, which the + # catch below then downgraded to a WARN while the + # code was deleted regardless. + $prevEap = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + & $Flask plugin uninstall $dir.Name --yes 2>&1 | + ForEach-Object { Write-Log " $_" } + $deregistered = ($LASTEXITCODE -eq 0) + } finally { $ErrorActionPreference = $prevEap } + } finally { Pop-Location } } } catch { Write-Log " could not deregister $($dir.Name): $($_.Exception.Message)" 'WARN' } - Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue + # Delete the code ONLY once the registry entry is really gone. + # Deleting it anyway left the application loading a plugin + # whose directory no longer existed - the exact failure the + # uninstall-first ordering above exists to avoid. + if ($deregistered) { + Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue + } else { + Write-Log " left $($dir.Name) on disk: deregistration did not succeed" 'WARN' + } } } } @@ -1039,6 +1392,34 @@ a page that cannot load its own assets. Rebuild with scripts/build-site.sh } # --- .env --------------------------------------------------------------- + # The stage-0 handoff is checked BEFORE the .env branch, not inside its else. + # It used to sit in the else, so it was consulted only when .env was absent - + # and on a part-finished install .env DOES exist, holding whatever password + # stage 2 last wrote. If stage 0 has since regenerated the credential, .env + # is stale, .dbpass is correct, and the installer preferred the stale one: + # "Access denied", with the right password sitting unread on disk. + # + # .dbpass only exists between stage 0 and stage 5 - stage 5 shreds it once + # the install is proven - so its presence means "this install is unfinished + # and THIS is the current password". Applied only when .env points at the + # local server, because the handoff describes the bundled database and must + # never redirect a site whose database lives elsewhere. + $handoff = Join-Path $AppRoot '.dbpass' + if (-not $DbPasswordFile -and (Test-Path $handoff)) { + $envIsLocal = $true + if (Test-Path $EnvFile) { + $envIsLocal = $false + $urlLine = Get-Content $EnvFile -ErrorAction SilentlyContinue | + Where-Object { $_ -like 'DATABASE_URL=*' } | Select-Object -First 1 + if ($urlLine -and ($urlLine -match '@(127\.0\.0\.1|localhost|\[::1\]):')) { $envIsLocal = $true } + } + if ($envIsLocal) { + $DbPasswordFile = $handoff + if (-not $DbHost) { $DbHost = '127.0.0.1' } + Write-Log 'using the stage-0 password handoff (an unfinished install; .env may be stale)' + } + } + if (Test-Path $EnvFile) { # Preserving .env is right for a plain re-run: regenerating JWT_SECRET_KEY # would invalidate every issued session. @@ -1131,17 +1512,7 @@ a page that cannot load its own assets. Rebuild with scripts/build-site.sh } else { if (-not $SiteHost) { $SiteHost = $env:COMPUTERNAME } - # Stage 0 leaves an ACL'd handoff file when it created the database itself. - # Prefer it: a greenfield install then needs no password from anyone, and - # the generated password never passes through a human or a command line. - # This MUST come before the -DbHost check below - stage 0 always installs - # to the local box, so the handoff implies the host as well as the password. - $handoff = Join-Path $AppRoot '.dbpass' - if (-not $DbPasswordFile -and (Test-Path $handoff)) { - $DbPasswordFile = $handoff - if (-not $DbHost) { $DbHost = '127.0.0.1' } - Write-Log 'using the bundled-MySQL password handoff from stage 0 (host 127.0.0.1)' - } + # The stage-0 handoff was already applied above, for both branches. if (-not $DbHost) { Fail 'no database host supplied and no existing .env' @' @@ -1311,9 +1682,22 @@ See deploy/site-profile.example.json for the format. # separately runnable ("re-run stage 3"), and under Set-StrictMode # reading an unset $script:IsUpgrade is a terminating error. $script:DbWasEmpty = Test-DatabaseEmpty + # Tables present does NOT mean upgrade. It meant that for a long time, + # and it is why the retry after a failed first install dead-ended: stage 3 + # had already created the schema before dying, so the retry found tables, + # called itself an upgrade of a live site, and then demanded a mandatory + # backup and refused to prune - of a database it had written itself. + # The install record distinguishes the two, which nothing on the machine + # can. + $firstRun = Test-FirstProvisioning if (-not $script:DbWasEmpty) { - $script:IsUpgrade = $true - Write-Log 'existing data found in the target database - treating this as an upgrade' 'WARN' + if ($firstRun) { + Write-Log 'the database already has tables, but the install record shows first provisioning is still unfinished' 'WARN' + Write-Log ' they were created by an earlier attempt of THIS install, so this is not an upgrade' 'WARN' + } else { + $script:IsUpgrade = $true + Write-Log 'existing data found in the target database - treating this as an upgrade' 'WARN' + } } # Back up BEFORE Alembic touches anything. Skipped only when the database @@ -1323,10 +1707,19 @@ See deploy/site-profile.example.json for the format. if (-not $script:DbWasEmpty) { $script:PreUpgradeBackup = Backup-Database 'pre-upgrade' if (-not $script:PreUpgradeBackup) { - Fail 'could not back up the database before upgrading' @' + if ($firstRun) { + # Nothing of anyone's to lose: every row was written by an + # earlier attempt of this same install. Refusing here turned a + # missing mysqldump into a hard dead end on exactly the retry + # the wizard tells the operator is safe to run. + Write-Log 'no pre-migration backup was taken' 'WARN' + Write-Log ' continuing anyway: this is still first provisioning, so the only data present is what an earlier attempt wrote' 'WARN' + } else { + Fail 'could not back up the database before upgrading' @' Refusing to migrate without a backup. Fix mysqldump access (or take a backup manually with shopdb-admin.ps1 backup) and run this again. '@ + } } } @@ -1404,19 +1797,41 @@ build to get a matching pair, then send the install log to support. # data for any plugin that is not installed - so never force there. # Without --force, prune-schema refuses non-empty tables and says so, # which is the behaviour we want when data exists. - if (-not $script:DbWasEmpty) { + # --force is safe during first provisioning for the same reason the backup + # is not required: every row present was written by an earlier attempt of + # this install. Core migration 7d05 seeds access protocols owned by the + # computers plugin, so any profile omitting computers hit the refusal on + # every single retry and could never get past stage 3. + if ((-not $script:DbWasEmpty) -and (-not $firstRun)) { Write-Log 'flask plugin prune-schema --yes (no --force: refuses to drop tables holding data)' - $pruneOut = Invoke-Native $Flask @('plugin','prune-schema','--yes') 'prune not-installed plugin tables' - $skipped = @($pruneOut | Where-Object { $_ -match 'refus|not empty|rows' }) + # -OkExit 0,1: refusing is the DESIGNED outcome here, and the CLI + # signals it with SystemExit(1). Without this, Invoke-Native failed + # the stage the moment prune declined to drop a table holding rows - + # and the reporting below, which exists precisely to explain that, + # could never run. Core migration 7d05 seeds access protocols owned + # by the computers plugin, so any profile omitting computers hit it + # on every retry. + $pruneOut = Invoke-Native $Flask @('plugin','prune-schema','--yes') ` + 'prune not-installed plugin tables' -OkExit 0,1 + $skipped = @($pruneOut | Where-Object { $_ -match 'REFUSING|rows$|not empty' }) if ($skipped.Count -gt 0) { Write-Log 'some plugin tables were kept because they hold data:' 'WARN' $skipped | ForEach-Object { Write-Log " $_" 'WARN' } Write-Log ' drop them by hand only if you are certain that data is not needed' 'WARN' + } elseif ($pruneOut -match 'not initialized') { + # The other reason this CLI exits 1, and it is a genuine failure. + Fail 'the plugin manager did not initialise' 'Check the application log; the app could not start.' } } else { - # Provably-empty database only: core migrations seed a few plugin - # reference tables, so prune would otherwise refuse. - Write-Log 'flask plugin prune-schema --yes --force (database was empty at start)' + # Reached when the database was provably empty, OR when the install + # record says first provisioning is still unfinished. Both mean the + # only rows present are ones this install put there - core migrations + # seed a few plugin reference tables, so prune would otherwise refuse. + if ($script:DbWasEmpty) { + Write-Log 'flask plugin prune-schema --yes --force (database was empty at start)' + } else { + Write-Log 'flask plugin prune-schema --yes --force (still first provisioning: these tables are ours)' + } Invoke-Native $Flask @('plugin','prune-schema','--yes','--force') 'prune not-installed plugin tables' } } @@ -1454,7 +1869,7 @@ the module by hand, then re-run stage 4. Write-Log "installing $($hphMsi.Name)" if (-not $WhatIfOnly) { Invoke-Native 'msiexec.exe' @('/i', $hphMsi.FullName, '/quiet', '/norestart') ` - 'HttpPlatformHandler MSI' -TimeoutSec 600 + 'HttpPlatformHandler MSI' -TimeoutSec 600 -OkExit 0,3010 if (-not (Test-Path $hphDll)) { Fail 'the HttpPlatformHandler MSI reported success but the module is missing' ` 'An installer exit code of 0 does not prove it did what was asked.' @@ -1486,7 +1901,7 @@ has to come from the bundle either way. Write-Log "installing $($rwMsi.Name)" if (-not $WhatIfOnly) { Invoke-Native 'msiexec.exe' @('/i', $rwMsi.FullName, '/quiet', '/norestart') ` - 'URL Rewrite MSI' -TimeoutSec 600 + 'URL Rewrite MSI' -TimeoutSec 600 -OkExit 0,3010 if (-not (Test-Path $rewriteDll)) { Fail 'the URL Rewrite MSI reported success but the module is missing' ` 'An installer exit code of 0 does not prove it did what was asked.' @@ -1511,6 +1926,40 @@ has to come from the bundle either way. if (Test-Path $dstCfg) { Write-Log 'web.config already exists; leaving it alone' 'OK' $existing = Get-Content $dstCfg -Raw + + # ONE EXCEPTION to leaving it alone. An block is + # fatal by itself: that section is Deny by default, so IIS rejects the + # whole file with 500.52 (0x80070021) before httpPlatformHandler runs. + # The app never starts and its stdout log stays empty, which reads as a + # broken application and is not one. + # + # Earlier builds of THIS installer wrote exactly that block, so honouring + # "leave it alone" without qualification would leave every server already + # installed permanently unable to start, with re-running powerless to fix + # it. Strip just that element and keep every other edit - and only when it + # contains nothing but the variable this installer puts there, because + # anything else is somebody's deliberate change. + if ($existing -match '(?s)(.*?)') { + $inner = $Matches[1] + $names = [regex]::Matches($inner, 'name\s*=\s*"([^"]+)"') | + ForEach-Object { $_.Groups[1].Value } + # @() around the pipeline: a zero-match filter returns $null, and + # $null.Count is a terminating error under Set-StrictMode 2.0. + $foreign = @($names | Where-Object { $_ -ne 'HTTP_X_FORWARDED_FOR' }).Count + if ($foreign -gt 0) { + Write-Log 'web.config declares holding variables this installer did not add' 'WARN' + Write-Log ' IIS returns 500.52 unless that section is unlocked server-wide; left unchanged' 'WARN' + } elseif (-not $WhatIfOnly) { + $backup = "$dstCfg.before-xff-fix" + Copy-Item $dstCfg $backup -Force + $fixed = $existing -replace '(?s)[ \t]*.*?\r?\n?', '' + Set-Content -Path $dstCfg -Value $fixed -Encoding UTF8 + Write-Log 'removed from web.config; it made IIS return 500.52' 'OK' + Write-Log " the previous file is kept at $backup" + $existing = $fixed + } + } + $hasRule = $existing -match '' -and $existing -notmatch 'SHOPDB-CLIENTIP-BEGIN' if ($ClientIpSource -eq 'direct' -and -not $hasRule) { Write-Log 'this web.config does NOT set X-Forwarded-For; client IPs will read as 127.0.0.1' 'WARN' @@ -1567,27 +2016,6 @@ has to come from the bundle either way. Invoke-Native 'icacls.exe' @($EnvFile,'/grant',"${ident}:(R)") 'ACL on .env' } - # Handler sections are locked server-wide by default. Without unlocking, - # IIS returns 500.19 the moment it reads the app's web.config. - # - # `appcmd unlock config /section:X` unlocks it for EVERY site on the machine, - # which on a shared server hands every other application the ability to - # define its own handlers. Delegate to our own location instead, and fall - # back to the server-wide unlock only if that is refused - a 500.19 the - # operator cannot diagnose is worse than a wider delegation, but it should be - # the second choice, not the first. - $ourLocation = if ($MountAlias) { "$ParentSite/$($MountAlias.Trim('/'))" } else { $SiteName } - foreach ($section in @('system.webServer/handlers','system.webServer/httpPlatform')) { - if ($WhatIfOnly) { Write-Log "would unlock $section for $ourLocation"; continue } - Write-Log "unlocking $section for $ourLocation" - $scoped = & $appcmd unlock config "$ourLocation" /section:$section 2>&1 - $scoped | ForEach-Object { Write-Log " $_" } - if ($LASTEXITCODE -ne 0) { - Write-Log " scoped unlock refused; unlocking $section server-wide" 'WARN' - & $appcmd unlock config /section:$section 2>&1 | ForEach-Object { Write-Log " $_" } - } - } - # Switching method must not leave BOTH deployments in place: two entry points # to one directory, one of them serving an SPA built for the wrong base path. # Remove whichever artifact belongs to the method we are NOT using. @@ -1617,7 +2045,7 @@ has to come from the bundle either way. } else { foreach ($site in (Get-Website)) { foreach ($app in (Get-WebApplication -Site $site.Name -ErrorAction SilentlyContinue)) { - if ($app.PhysicalPath -eq $AppRoot) { + if (Test-SamePath $app.PhysicalPath $AppRoot) { $doomed += @{ Kind = 'app'; Name = $app.Path.Trim('/'); Site = $site.Name } } } @@ -1683,6 +2111,36 @@ If not, re-run matching how the application is published today: pass # METHOD A: its own site on $SitePort. if (Get-Website -Name $SiteName -ErrorAction SilentlyContinue) { Write-Log "site $SiteName already exists" 'OK' + # RECONCILE, do not just accept it. "Already exists" used to take the + # site however it was, so re-running with a different port left the + # binding alone while everything downstream - CORS_ORIGINS, the + # firewall rule, the shortcut, the stage 5 smoke test - used the NEW + # port. A working server was then reported as a failed install. + # + # Refuse rather than silently re-bind: moving where people reach the + # site is not something to do because a field defaulted. + if (-not $WhatIfOnly) { + $existingPorts = @() + $existingPath = '' + try { + $siteNow = Get-Website -Name $SiteName + $existingPath = [string] $siteNow.physicalPath + foreach ($b in $siteNow.bindings.Collection) { + if ($b.protocol -eq 'http' -and ($b.bindingInformation -match '^[^:]*:(\d+):')) { + $existingPorts += [int] $Matches[1] + } + } + } catch { } + if (($existingPorts.Count -gt 0) -and ($existingPorts -notcontains [int] $SitePort)) { + Fail ("site '{0}' is bound to port {1}, but this run was told to use {2}" -f ` + $SiteName, ($existingPorts -join ', '), $SitePort) ` + 'Re-run and enter the port the site already uses, or change the binding in IIS Manager first.' + } + if ($existingPath -and (-not (Test-SamePath $existingPath $AppRoot))) { + Fail ("site '{0}' serves {1}, not {2}" -f $SiteName, $existingPath, $AppRoot) ` + 'Something else owns that site name. Install into the directory it serves, or use a different site name.' + } + } } else { Write-Log "creating site $SiteName on port $SitePort" if (-not $WhatIfOnly) { @@ -1705,6 +2163,82 @@ If not, re-run matching how the application is published today: pass if (-not $WhatIfOnly) { Start-Website -Name $SiteName -ErrorAction SilentlyContinue } } + + # Handler sections are locked server-wide by default. Without unlocking, + # IIS returns 500.19 the moment it reads the app's web.config. + # + # `appcmd unlock config /section:X` unlocks it for EVERY site on the machine, + # which on a shared server hands every other application the ability to + # define its own handlers. Delegate to our own location instead, and fall + # back to the server-wide unlock only if that is refused - a 500.19 the + # operator cannot diagnose is worse than a wider delegation, but it should be + # the second choice, not the first. + # + # MUST run AFTER the site or application exists. appcmd resolves the location + # against applicationHost.config, so unlocking "Default Web Site/shopdb" + # before New-WebApplication created it failed with 80070003 ("the system + # cannot find the path specified") on every first install - and the fallback + # then handed handler delegation to the whole machine, which is exactly the + # outcome the comment above calls the second choice. A first install could + # never take the scoped path. + $ourLocation = if ($MountAlias) { "$ParentSite/$($MountAlias.Trim('/'))" } else { $SiteName } + foreach ($section in @('system.webServer/handlers','system.webServer/httpPlatform')) { + if ($WhatIfOnly) { Write-Log "would unlock $section for $ourLocation"; continue } + Write-Log "unlocking $section for $ourLocation" + # EAP must drop to Continue around these. appcmd writes to stderr on + # ordinary refusals, and '2>&1' with $ErrorActionPreference = 'Stop' + # turns that into a TERMINATING NativeCommandError - so the exit-code + # test and the server-wide fallback below, which is the entire point of + # the block, were unreachable, and stage 4 aborted after Python, the + # venv, the schema and the ACLs had all been changed. + $prevEap = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $scoped = & $appcmd unlock config "$ourLocation" /section:$section 2>&1 + $scoped | ForEach-Object { Write-Log " $_" } + if ($LASTEXITCODE -ne 0) { + Write-Log " scoped unlock refused; unlocking $section server-wide" 'WARN' + & $appcmd unlock config /section:$section 2>&1 | ForEach-Object { Write-Log " $_" } + if ($LASTEXITCODE -ne 0) { + Write-Log " server-wide unlock also refused - IIS may return 500.19" 'WARN' + } + } + } finally { $ErrorActionPreference = $prevEap } + } + + # URL Rewrite: a rule may only set a server variable that is ALLOWED, and + # system.webServer/rewrite/allowedServerVariables ships with + # overrideModeDefault="Deny". An block in the app's + # own web.config is therefore refused outright - IIS answered 500.52 with + # 0x80070021, "this configuration section cannot be used at this path", + # before it ever reached httpPlatformHandler. python was never launched, the + # stdout log stayed empty, and the stage 5 smoke test reported it as a dead + # application when the config had been rejected wholesale. + # + # Allow the ONE variable at server level rather than unlocking the section. + # Unlocking would let every site on this machine declare arbitrary server + # variables; this grants exactly HTTP_X_FORWARDED_FOR. + if (($ClientIpSource -eq 'direct') -and (-not $WhatIfOnly)) { + $varName = 'HTTP_X_FORWARDED_FOR' + $section = 'system.webServer/rewrite/allowedServerVariables' + $prevEap = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + # Adding a duplicate entry is an error, so look before writing. + $current = (& $appcmd list config /section:$section 2>&1 | Out-String) + if ($current -match [regex]::Escape($varName)) { + Write-Log "$varName is already allowed for URL Rewrite" 'OK' + } else { + Write-Log "allowing $varName for URL Rewrite (server level)" + & $appcmd set config /section:$section "/+[name='$varName']" /commit:apphost 2>&1 | + ForEach-Object { Write-Log " $_" } + if ($LASTEXITCODE -ne 0) { + Write-Log " could not allow $varName - IIS will return 500.52 and the app will not start" 'WARN' + } + } + } finally { $ErrorActionPreference = $prevEap } + } + # Stage 2 stops the pool on an upgrade so its files can be replaced. Nothing # else starts it again, so the smoke test would fail against a stopped site # and report it as a broken install. @@ -1803,15 +2337,66 @@ public class ShopdbSmokeTestCertPolicy : ICertificatePolicy { # First request boots the app and connects to MySQL: the runbook records ~15s. Write-Log "requesting $($targets[0]) (first request takes ~15s while the app boots)" $ok = $false + $lastStatus = 'no response' + $lastDetail = '' for ($i = 1; $i -le 12; $i++) { try { $r = Invoke-WebRequest -Uri $targets[0] -UseBasicParsing -TimeoutSec 20 + $lastStatus = "HTTP $($r.StatusCode)" if ($r.StatusCode -eq 200) { $ok = $true; break } + # A non-200 that does NOT throw (a redirect, say) is still not success. + # Without this sleep the loop burned all 12 attempts instantly and + # reported a timeout that never waited for anything. + Start-Sleep -Seconds 5 } catch { + # The status code and the response body ARE the diagnosis, and this + # block used to discard both - leaving a generic "check the logs" for + # a fault IIS had already named. 500.19 = section still locked, + # 502.3/503 = httpPlatform could not start the app, 404 = the + # application or its handler mapping is missing. + $lastStatus = 'no response' + $lastDetail = $_.Exception.Message + try { + $resp = $_.Exception.Response + if ($resp) { + $lastStatus = "HTTP {0} ({1})" -f [int]$resp.StatusCode, $resp.StatusCode + $reader = New-Object System.IO.StreamReader($resp.GetResponseStream()) + $body = $reader.ReadToEnd() + $reader.Close() + if ($body) { + # IIS error pages state the precise cause in their body; + # strip the markup so it fits in the log readably. + $clean = (($body -replace '<[^>]+>', ' ') -replace '\s+', ' ').Trim() + if ($clean) { + if ($clean.Length -gt 500) { $clean = $clean.Substring(0, 500) + ' ...' } + $lastDetail = $clean + } + } + } + } catch { } Start-Sleep -Seconds 5 } } if (-not $ok) { + Write-Log " last response: $lastStatus" 'FAIL' + if ($lastDetail) { Write-Log " $lastDetail" 'FAIL' } + # The HttpPlatform stdout log is where a Python traceback lands. Reading it + # here saves a round trip: whoever is sending this log back already has the + # cause in it, instead of being told which second file to go and open. + try { + $logDir = Join-Path $AppRoot 'logs' + $newest = Get-ChildItem $logDir -Filter '*.log' -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | Select-Object -First 1 + if (-not $newest) { + Write-Log " no HttpPlatform log in $logDir - the pool never launched python" 'FAIL' + } elseif ($newest.Length -eq 0) { + Write-Log " $($newest.Name) is empty - python was launched but wrote nothing" 'FAIL' + } else { + Write-Log " tail of $($newest.Name):" 'FAIL' + Get-Content $newest.FullName -Tail 20 -ErrorAction SilentlyContinue | + ForEach-Object { Write-Log " $_" 'FAIL' } + } + } catch { } Fail "site did not return 200 at $($targets[0])" ` "Check $AppRoot\logs. Empty HttpPlatform log usually means the app-pool identity cannot read $AppRoot or run the venv, or .env is missing/invalid." } @@ -1998,11 +2583,19 @@ function Invoke-Uninstall { $removed = $false foreach ($site in (Get-Website -ErrorAction SilentlyContinue)) { $app = Get-WebApplication -Site $site.Name -Name $mount -ErrorAction SilentlyContinue - if ($app) { - Remove-WebApplication -Site $site.Name -Name $mount -ErrorAction SilentlyContinue - Write-Log "removed application /$mount from site '$($site.Name)'" 'OK' - $removed = $true + if (-not $app) { continue } + # OWNERSHIP CHECK. Matching on the alias alone removed ANY application + # on the server that happened to share the name - a second instance, + # or an unrelated /shopdb under a different site - and -OnFailure + # never suppresses the typed confirmation, so it happened unattended. + if (-not (Test-SamePath $app.PhysicalPath $AppRoot)) { + Write-Log ("left /{0} under '{1}' alone: it serves {2}, not {3}" -f ` + $mount, $site.Name, $app.PhysicalPath, $AppRoot) 'WARN' + continue } + Remove-WebApplication -Site $site.Name -Name $mount -ErrorAction SilentlyContinue + Write-Log "removed application /$mount from site '$($site.Name)'" 'OK' + $removed = $true } if (-not $removed) { Write-Log "application /$mount not present" } } @@ -2071,6 +2664,11 @@ Write-Log "ShopDB-Flask installer, stage=$Stage, approot=$AppRoot, bundle=$Bundl if ($WhatIfOnly) { Write-Log 'WhatIfOnly: no changes will be made' 'WARN' } try { + # Before ANY stage, so that a stage which dies still leaves a record saying + # provisioning had started. Skipped for uninstall, which must not create a + # record of an install it is removing. + if ($Stage -ne 'uninstall') { Initialize-InstallState } + switch ($Stage) { '0' { Invoke-Stage0 } '1' { & (Join-Path $PSScriptRoot 'shopdb-preflight.ps1') -SitePort $SitePort -AppRoot $AppRoot } @@ -2083,6 +2681,11 @@ try { 'all' { Invoke-Stage2; Invoke-Stage3; Invoke-Stage4; Invoke-Stage5 } 'uninstall' { Invoke-Uninstall } } + # Provisioning is finished only when a run that INCLUDED the later stages + # completed. A successful stage 2 on its own leaves first-run open, because + # the schema and the site are still ahead of it, and the retry that follows + # must still be treated as part of the first install. + if ($Stage -eq 'all' -or $Stage -eq '5') { Complete-InstallState } Write-Log "done. log: $script:LogPath" 'OK' exit 0 } @@ -2099,6 +2702,20 @@ catch { if ($doRollback) { Invoke-Rollback } else { Write-Log 'rollback declined; partial install left in place' 'WARN' } } + # Stage 2 stops the pool on an upgrade so files can be replaced. If the run + # then failed, a site that was SERVING before this started is now off, and + # saying only "part-configured" left the operator to discover that from users. + # Not restarted automatically: after a failure in or past stage 3 the deployed + # code and the schema may no longer agree, and starting it would put a broken + # application back in front of people. Say it plainly instead. + $poolStopped = $false + try { $poolStopped = [bool] $script:PoolWasStopped } catch { } + if ($poolStopped) { + Write-Log "the application pool '$AppPool' was stopped for this upgrade and is STILL STOPPED" 'FAIL' + Write-Log ' the site was serving before this run and is now offline' 'FAIL' + Write-Log " once the cause above is fixed, re-run this installer, or start it by hand with:" 'FAIL' + Write-Log " Start-WebAppPool -Name '$AppPool'" 'FAIL' + } Write-Log "log: $script:LogPath" 'FAIL' exit 1 } diff --git a/deploy/windows/installer/shopdb-preflight.ps1 b/deploy/windows/installer/shopdb-preflight.ps1 index dfaaeee6..75559da2 100644 --- a/deploy/windows/installer/shopdb-preflight.ps1 +++ b/deploy/windows/installer/shopdb-preflight.ps1 @@ -221,8 +221,12 @@ Invoke-Check 'Network' 'Site port' { Add-Result 'Network' 'Site port' 'INFO' ` "TCP $SitePort is used by the existing $SiteName site - this will be upgraded in place" } else { - Add-Result 'Network' 'Site port' 'FAIL' "TCP $SitePort is in use$owner" ` - "Choose a different port with -SitePort, or stop the listener." + # WARN, not FAIL. This check runs before the operator has reached the + # Address page, so it is testing the DEFAULT port, not necessarily + # the one they intend to use. Blocking here would refuse an install + # over a conflict the very next page lets them resolve. + Add-Result 'Network' 'Site port' 'WARN' "TCP $SitePort is in use$owner" ` + "Pick a different port on the Address page later in this wizard, or stop whatever is holding it." } } } @@ -266,8 +270,13 @@ Invoke-Check 'IIS' 'HttpPlatformHandler' { if ($modules -match 'httpPlatformHandler') { Add-Result 'IIS' 'HttpPlatformHandler' 'PASS' 'installed' } else { - Add-Result 'IIS' 'HttpPlatformHandler' 'FAIL' 'not installed' ` - 'Install httpPlatformHandler_amd64.msi from the bundle. IIS cannot launch waitress without it.' + # NOT a blocker: the MSI is in the bundle and stage 4 installs it. This + # was a FAIL, which - once the preflight page started blocking on any + # failure - stopped the wizard dead over something the installer was + # about to do by itself, with no way forward but to go and install it by + # hand. Nothing the installer SUPPLIES may be a blocker. + Add-Result 'IIS' 'HttpPlatformHandler' 'INFO' 'not installed yet' ` + 'The installer installs it from the bundle. No action needed.' } } @@ -350,7 +359,7 @@ Invoke-Check 'MySQL' 'Service' { $svcs = @(Get-Service -ErrorAction SilentlyContinue | Where-Object { $_.Name -match '^MySQL' -or $_.DisplayName -match 'MySQL' }) if ($svcs.Count -eq 0) { Add-Result 'MySQL' 'Service' 'INFO' 'no MySQL service found' ` - 'Bundled MySQL 8.0 is the appropriate choice on this box.' + 'Bundled MySQL 8.4 LTS is the appropriate choice on this box.' return } foreach ($s in $svcs) { @@ -378,7 +387,7 @@ Invoke-Check 'MySQL' 'Backup client' { $found = '' foreach ($root in @((Join-Path $PSScriptRoot 'mysqlclient'), (Join-Path $AppRoot 'mysqlclient'), - 'C:\Program Files\MySQL', 'C:\mysql56\bin', + 'C:\MySQL84', 'C:\Program Files\MySQL', 'C:\mysql56\bin', 'C:\Program Files (x86)\MySQL')) { if (-not (Test-Path $root)) { continue } $hit = Get-ChildItem $root -Filter $names[0] -Recurse -ErrorAction SilentlyContinue | @@ -389,7 +398,7 @@ Invoke-Check 'MySQL' 'Backup client' { Add-Result 'MySQL' 'Backup client' 'PASS' "mysqldump found ($found)" } else { Add-Result 'MySQL' 'Backup client' 'WARN' 'mysqldump not found on this server' ` - 'Needed for the automatic pre-upgrade backup and for "shopdb-admin.ps1 backup". A first install works without it; upgrades will not be protected. Add mysqlclient\ to the installer bundle, or install the MySQL client on this server.' + 'Needed for the automatic pre-upgrade backup and for "shopdb-admin.ps1 backup". A first install works without it; upgrades will not be protected. It is not on this server yet - the installer stages its own copy, so this normally resolves itself during installation.' } } @@ -430,7 +439,13 @@ Invoke-Check 'MySQL' 'Version and config' { Add-Result 'MySQL' '5.6 index flags' 'WARN' 'all three present in my.ini' ` 'Present in the file only. They take effect after a MySQL RESTART, which interrupts the classic ASP app. Confirm with SHOW VARIABLES or flask db-utils preflight.' } else { - Add-Result 'MySQL' '5.6 index flags' 'FAIL' ("missing: " + ($missing -join ', ')) ` + # WARN, not FAIL. This inspects the LOCAL MySQL, which may not be the + # database the operator is about to install against - a bundled 8.4, + # or a remote server. Blocking the wizard here refused an install + # over a server that had nothing to do with it. Stage 3 runs + # 'flask db-utils preflight' against the database actually chosen, + # which is the check that can genuinely block. + Add-Result 'MySQL' '5.6 index flags' 'WARN' ("missing: " + ($missing -join ', ')) ` "Add to [mysqld] in $ini and restart MySQL, or 'flask db upgrade' fails with error 1071. NOTE: restarting interrupts the classic ASP app." } } diff --git a/deploy/windows/installer/tests/test-install-state.ps1 b/deploy/windows/installer/tests/test-install-state.ps1 new file mode 100644 index 00000000..c5e19dcf --- /dev/null +++ b/deploy/windows/installer/tests/test-install-state.ps1 @@ -0,0 +1,130 @@ +# Tests the installer's durable install record. +# +# The record answers one question: "is this a re-run of MY install?". Getting it +# wrong is destructive in one direction - answering "first provisioning" for a +# server holding real data lets prune-schema --force loose on it - and a dead +# end in the other, which is what it was built to fix. +# +# Run directly: pwsh -File test-install-state.ps1 +# pytest runs it through tests/test_installer_state.py wherever pwsh exists. + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = 'Stop' + +$Here = Split-Path -Parent $MyInvocation.MyCommand.Path +$Installer = Join-Path (Split-Path -Parent $Here) 'shopdb-install.ps1' +if (-not (Test-Path $Installer)) { throw "installer not found at $Installer" } + +$Sandbox = Join-Path ([System.IO.Path]::GetTempPath()) ('shopdb-state-' + [System.Guid]::NewGuid().ToString('N')) + +# Load ONLY the state block, so the rest of the installer does not run. Bounded +# by the same markers the installer uses, and it fails loudly if either moves +# rather than silently testing nothing. +$lines = Get-Content $Installer +$start = ($lines | Select-String -Pattern '^\$script:StateFile ' | Select-Object -First 1) +$end = ($lines | Select-String -Pattern '^function Invoke-Native' | Select-Object -First 1) +if (-not $start -or -not $end) { throw 'could not locate the install-record block in shopdb-install.ps1' } +$block = $lines[($start.LineNumber - 1)..($end.LineNumber - 2)] -join "`n" +if ($block -notmatch 'function Test-FirstProvisioning') { throw 'extracted block does not contain the state functions' } + +# Stand-ins for the surrounding script. +$script:Created = New-Object System.Collections.ArrayList +function Write-Log { param($m, $l = 'INFO') } +function Protect-File { param([string] $Path, [switch] $Directory) } +function Test-SamePath { + param([string] $A, [string] $B) + if (-not $A) { return -not $B } + return ($A.TrimEnd('\', '/').ToLowerInvariant() -eq $B.TrimEnd('\', '/').ToLowerInvariant()) +} +if (-not $env:ProgramData) { $env:ProgramData = [System.IO.Path]::GetTempPath() } + +Invoke-Expression $block + +$script:Failures = 0 +function Check { + param([string] $Name, $Expected, $Actual) + if ($Expected -eq $Actual) { + Write-Host (" PASS " + $Name) + } else { + Write-Host (" FAIL {0}: expected [{1}], got [{2}]" -f $Name, $Expected, $Actual) + $script:Failures++ + } +} + +function Reset-Case { + param([switch] $Stamped) + if (Test-Path $Sandbox) { Remove-Item $Sandbox -Recurse -Force } + New-Item -ItemType Directory -Path $Sandbox -Force | Out-Null + $script:AppRoot = $Sandbox + $script:StateFile = Join-Path $Sandbox 'install-state.json' + $script:InstallState = $null + if ($Stamped) { Set-Content (Join-Path $Sandbox '.installed-version') '0.7.0' } +} + +Write-Host 'brand new server' +Reset-Case +Initialize-InstallState +Check 'first provisioning' $true (Test-FirstProvisioning) +Check 'record written' $true (Test-Path $script:StateFile) + +Write-Host 'retry of a failed first install' +$script:InstallState = $null +Initialize-InstallState +Check 'still first provisioning' $true (Test-FirstProvisioning) + +Write-Host 'genuine upgrade after a completed install' +Complete-InstallState +$script:InstallState = $null +Initialize-InstallState +Check 'no longer first provisioning' $false (Test-FirstProvisioning) + +Write-Host 'install predating the record - must take the safe side' +Reset-Case -Stamped +Initialize-InstallState +Check 'treated as established' $false (Test-FirstProvisioning) + +Write-Host 'record naming a different directory' +Reset-Case +Initialize-InstallState +$other = Get-Content $script:StateFile -Raw | ConvertFrom-Json +$other.approot = 'C:\somewhere-else' +($other | ConvertTo-Json -Depth 6) | Set-Content $script:StateFile +$script:InstallState = $null +Check 'foreign record rejected' $null (Get-InstallState) + +Write-Host 'corrupt record' +Reset-Case +Set-Content $script:StateFile '{ this is not json' +$script:InstallState = $null +Check 'corrupt record rejected' $null (Get-InstallState) +Reset-Case +Set-Content $script:StateFile '{ this is not json' +Initialize-InstallState +Check 'recovers and rewrites' $true ($null -ne $script:InstallState) + +Write-Host 'created items survive a crash' +Reset-Case +Initialize-InstallState +Track 'site' 'shopdb-flask' +Track 'apppool' 'shopdbflask' +Track 'site' 'shopdb-flask' +$script:InstallState = $null +Initialize-InstallState +Check 'site remembered' 'shopdb-flask' ((Get-CreatedItems 'site') -join ',') +Check 'apppool remembered' 'shopdbflask' ((Get-CreatedItems 'apppool') -join ',') +# A zero-length array returned from a function unrolls to $null, and $null.Count +# is fatal under StrictMode. This is the check that catches losing the comma. +Check 'unknown kind is an empty array' 0 ((Get-CreatedItems 'firewall')).Count + +Write-Host 'no record at all' +$script:InstallState = $null +Check 'empty list, not null' 0 ((Get-CreatedItems 'site')).Count +Check 'not first provisioning' $false (Test-FirstProvisioning) + +if (Test-Path $Sandbox) { Remove-Item $Sandbox -Recurse -Force } +if ($script:Failures -gt 0) { + Write-Host ("{0} check(s) failed" -f $script:Failures) + exit 1 +} +Write-Host 'all checks passed' +exit 0 diff --git a/deploy/windows/installer/verify_bundle_lock.py b/deploy/windows/installer/verify_bundle_lock.py index 80634f09..b8d036a3 100644 --- a/deploy/windows/installer/verify_bundle_lock.py +++ b/deploy/windows/installer/verify_bundle_lock.py @@ -32,6 +32,7 @@ PAYLOADS = [ ('httpplatformhandler', True, 'the IIS module that launches waitress'), ('urlrewrite', False, 'IIS URL Rewrite, for the client-IP rule'), ('mysqlclient', False, 'mysql/mysqldump, for backups against a remote database'), + ('vcredist', False, 'the Visual C++ runtime MySQL requires'), ('mysql', False, 'MySQL, for the bundled-database option'), ] diff --git a/deploy/windows/shopdb-diagnose.py b/deploy/windows/shopdb-diagnose.py new file mode 100644 index 00000000..eaf63e51 --- /dev/null +++ b/deploy/windows/shopdb-diagnose.py @@ -0,0 +1,422 @@ +"""Collect everything needed to diagnose a ShopDB-Flask stage 5 failure. + +Stage 5 is the smoke test: the installer asks IIS for the site and expects 200. +When it does not get one, the cause is always in one of four places, and this +script reads all four in one pass so the answer arrives in a single round trip: + + 1. What IIS actually answers, and with which status code and error page. + 2. Whether the config sections httpPlatformHandler needs are unlocked. + 3. Whether the app-pool identity can read the app and run its venv. + 4. Whether the app itself imports and starts. + +Run it ON THE SERVER, as Administrator: + + C:\\Python314\\python.exe shopdb-diagnose.py + +It writes shopdb-diagnose-.txt next to itself and prints the path. +Send that file back. + +SECRETS: the report never contains them. Values from .env (database password, +SECRET_KEY, JWT_SECRET_KEY) are read first, then scrubbed out of every section +of the report before it is written, including command output and tracebacks +that might quote them. + +Standard library only, so it runs on the bundled runtime or any system Python. +""" + +import os +import re +import socket +import subprocess +import sys +import time +from datetime import datetime + +APP_ROOT = os.environ.get('SHOPDB_APPROOT', r'C:\shopdb-flask') +ALIAS = 'shopdb' +TIMEOUT = 25 + +# Filled from .env, then scrubbed from the whole report. +SECRETS = [] + +WINDIR = os.environ.get('WINDIR', r'C:\Windows') +# Sysnative gives a 32-bit process the real 64-bit System32. Harmless on 64-bit. +APPCMD_CANDIDATES = [ + os.path.join(WINDIR, 'Sysnative', 'inetsrv', 'appcmd.exe'), + os.path.join(WINDIR, 'System32', 'inetsrv', 'appcmd.exe'), +] + + +def find_appcmd(): + for path in APPCMD_CANDIDATES: + if os.path.isfile(path): + return path + return None + + +class Report(object): + def __init__(self): + self.chunks = [] + + def head(self, title): + self.chunks.append('\n' + '=' * 72 + '\n' + title + '\n' + '=' * 72) + + def line(self, text=''): + self.chunks.append(str(text)) + + def block(self, title, body): + self.chunks.append('\n--- %s ---' % title) + if body is None or str(body).strip() == '': + self.chunks.append('(no output)') + else: + self.chunks.append(str(body).rstrip()) + + def text(self): + return '\n'.join(self.chunks) + '\n' + + +def run(cmd, timeout=60): + """Run a command, return combined output. Never raises.""" + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=False) + out, _ = proc.communicate(timeout=timeout) + text = out.decode('utf-8', 'replace') if out else '' + return '[exit %s]\n%s' % (proc.returncode, text) + except subprocess.TimeoutExpired: + try: + proc.kill() + except Exception: + pass + return '[TIMED OUT after %ss]' % timeout + except Exception as exc: + return '[could not run: %s]' % exc + + +def powershell(script, timeout=90): + exe = os.path.join(WINDIR, 'Sysnative', 'WindowsPowerShell', 'v1.0', 'powershell.exe') + if not os.path.isfile(exe): + exe = 'powershell.exe' + return run([exe, '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', script], + timeout=timeout) + + +def load_secrets(): + """Read .env so its secret VALUES can be scrubbed from the report.""" + env_path = os.path.join(APP_ROOT, '.env') + found = {} + if not os.path.isfile(env_path): + return found, None + try: + with open(env_path, 'r', encoding='utf-8', errors='replace') as handle: + raw = handle.read() + except Exception as exc: + return found, '[could not read .env: %s]' % exc + + for line in raw.splitlines(): + line = line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + key, value = line.split('=', 1) + key, value = key.strip(), value.strip().strip('"').strip("'") + found[key] = value + if not value: + continue + upper = key.upper() + if 'SECRET' in upper or 'PASSWORD' in upper or 'TOKEN' in upper or 'KEY' in upper: + SECRETS.append(value) + if upper == 'DATABASE_URL': + # mysql+pymysql://user:PASSWORD@host/db -- the password only. + match = re.match(r'^[^:]+://([^:@/]+):([^@]+)@', value) + if match: + SECRETS.append(match.group(2)) + return found, raw + + +def scrub(text): + """Remove every known secret value from the report.""" + for secret in SECRETS: + if secret and len(secret) >= 4: + text = text.replace(secret, '') + # Catch a password inside any URL that did not come from .env. + text = re.sub(r'(://[^:@/\s]+:)[^@\s]+(@)', r'\1\2', text) + return text + + +def http_probe(url): + """Fetch a URL, returning status, headers and body even for an error page.""" + import urllib.error + import urllib.request + + started = time.time() + try: + request = urllib.request.Request(url, headers={'User-Agent': 'shopdb-diagnose'}) + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: + body = response.read(4000).decode('utf-8', 'replace') + return {'status': response.status, 'reason': response.reason, + 'headers': dict(response.headers), 'body': body, + 'seconds': time.time() - started} + except urllib.error.HTTPError as exc: + body = '' + try: + body = exc.read(4000).decode('utf-8', 'replace') + except Exception: + pass + return {'status': exc.code, 'reason': exc.reason, + 'headers': dict(exc.headers or {}), 'body': body, + 'seconds': time.time() - started} + except Exception as exc: + return {'status': None, 'reason': '%s: %s' % (type(exc).__name__, exc), + 'headers': {}, 'body': '', 'seconds': time.time() - started} + + +def summarise_iis_error(body): + """Pull the meaningful bits out of an IIS error page.""" + if not body: + return None + import html + + # style and script blocks first: their contents survive plain tag stripping + # and drag CSS into the summary. + flat = re.sub(r'(?is)<(script|style)[^>]*>.*?', ' ', body) + flat = re.sub(r'(?s)', ' ', flat) + flat = re.sub(r'(?s)<[^>]*>', ' ', flat) + # Any unterminated tag left by the 4000-byte body truncation. + flat = re.sub(r'(?s)<[^>]*$', ' ', flat) + flat = html.unescape(flat) + flat = re.sub(r'\s+', ' ', flat).strip() + + hints = [] + for pattern in (r'\b\d{3}\.\d+\b', r'0x[0-9a-fA-F]{8}', + r'Error Code[^.]{0,80}', r'Config (?:Error|File)[^.]{0,120}', + r'Requested URL[^.]{0,120}', r'Physical Path[^.]{0,120}'): + for match in re.findall(pattern, flat): + cleaned = re.sub(r'\s+', ' ', match).strip(' :-') + if cleaned and cleaned not in hints: + hints.append(cleaned) + return {'flat': flat[:1200], 'hints': hints} + + +def main(): + report = Report() + stamp = datetime.now().strftime('%Y%m%d-%H%M%S') + + env_values, env_raw = load_secrets() + + report.line('ShopDB-Flask stage 5 diagnostic') + report.line('generated %s' % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + report.line('host %s' % socket.gethostname()) + report.line('app root %s' % APP_ROOT) + report.line('python %s' % sys.version.replace('\n', ' ')) + report.line('process is %d-bit' % (64 if sys.maxsize > 2 ** 32 else 32)) + + # ---------------------------------------------------------------- 1. HTTP + report.head('1. WHAT IIS ANSWERS') + report.line('This is the single most important section. The status code names') + report.line('the fault: 500.19 = config locked, 502.3/503 = the app did not') + report.line('start, 404 = application or handler mapping missing.') + + hostname = socket.gethostname() + targets = [ + 'http://localhost/%s/' % ALIAS, + 'http://127.0.0.1/%s/' % ALIAS, + 'http://[::1]/%s/' % ALIAS, + 'http://%s/%s/' % (hostname, ALIAS), + 'http://localhost/', + ] + for url in targets: + result = http_probe(url) + report.line('\n%s' % url) + report.line(' status : %s %s' % (result['status'], result['reason'])) + report.line(' time : %.1fs' % result['seconds']) + server = result['headers'].get('Server') + if server: + report.line(' server : %s' % server) + summary = summarise_iis_error(result['body']) + if summary and summary['hints']: + report.line(' hints : %s' % ' | '.join(summary['hints'][:8])) + if summary and summary['flat']: + report.line(' body : %s' % summary['flat'][:600]) + + # localhost resolving to ::1 first has bitten this install before. + report.block('name resolution for localhost', run( + ['nslookup', 'localhost'], timeout=20)) + + # ------------------------------------------------------------- 2. IIS state + report.head('2. IIS STATE') + appcmd = find_appcmd() + if not appcmd: + report.line('appcmd.exe NOT FOUND - is the IIS role installed?') + else: + report.line('appcmd: %s' % appcmd) + report.block('sites', run([appcmd, 'list', 'sites'])) + report.block('applications', run([appcmd, 'list', 'apps'])) + report.block('app pools', run([appcmd, 'list', 'apppools'])) + report.block('worker processes (empty means nothing is running)', + run([appcmd, 'list', 'wp'])) + report.block('modules: httpPlatformHandler present?', + run([appcmd, 'list', 'modules'])) + # overrideMode tells us whether the unlock actually took effect. + # + # allowedServerVariables is in this list because it caused a 500.52 that + # the first two sections could not explain: it is Deny by default, so an + # block in the app's web.config is rejected + # before httpPlatformHandler runs. Checking only the sections we unlock + # would have missed the one we do not. + for section in ('system.webServer/handlers', + 'system.webServer/httpPlatform', + 'system.webServer/rewrite/allowedServerVariables', + 'system.webServer/rewrite/rules'): + report.block('lock state of %s' % section, + run([appcmd, 'list', 'config', '/section:%s' % section, + '/text:*'])) + + report.block('W3SVC / WAS services', powershell( + "Get-Service W3SVC,WAS | Format-Table Name,Status,StartType -AutoSize | Out-String")) + report.block('listeners on port 80', powershell( + "Get-NetTCPConnection -LocalPort 80 -State Listen -EA SilentlyContinue | " + "Format-Table LocalAddress,LocalPort,OwningProcess -AutoSize | Out-String")) + report.block('app pool detail', powershell( + "Import-Module WebAdministration -EA SilentlyContinue; " + "Get-Item IIS:\\AppPools\\shopdbflask -EA SilentlyContinue | " + "Select-Object name,state,managedRuntimeVersion,enable32BitAppOnWin64," + "@{n='identity';e={$_.processModel.identityType}} | Format-List | Out-String")) + + # --------------------------------------------------------- 3. app + config + report.head('3. APPLICATION AND CONFIG') + + web_config = os.path.join(APP_ROOT, 'web.config') + if os.path.isfile(web_config): + try: + with open(web_config, 'r', encoding='utf-8', errors='replace') as handle: + report.block('web.config', handle.read()) + except Exception as exc: + report.block('web.config', '[could not read: %s]' % exc) + else: + report.block('web.config', 'MISSING at %s' % web_config) + + if env_raw is None: + report.block('.env', 'MISSING at %s' % os.path.join(APP_ROOT, '.env')) + else: + # Keys and non-secret values only. Secret values are scrubbed anyway. + lines = [] + for key in sorted(env_values): + upper = key.upper() + secretish = ('SECRET' in upper or 'PASSWORD' in upper + or 'TOKEN' in upper or 'KEY' in upper + or upper == 'DATABASE_URL') + if secretish: + lines.append('%s = ' % (key, len(env_values[key]))) + else: + lines.append('%s = %s' % (key, env_values[key])) + report.block('.env (secret values withheld)', '\n'.join(lines)) + + venv_python = os.path.join(APP_ROOT, 'venv', 'Scripts', 'python.exe') + report.line('\nvenv python exists: %s' % os.path.isfile(venv_python)) + if os.path.isfile(venv_python): + # The exact failure stage 3 used to hit. Proves the app imports. + report.block('venv: import shopdb', run( + [venv_python, '-c', + 'import shopdb; print("import OK"); ' + 'app = shopdb.create_app(); print("create_app OK")'], timeout=120)) + report.block('venv: waitress present', run( + [venv_python, '-c', 'import waitress; print(waitress.__version__)'], + timeout=60)) + + # What httpPlatformHandler is told to launch, and whether it exists. + if os.path.isfile(web_config): + try: + with open(web_config, 'r', encoding='utf-8', errors='replace') as handle: + raw = handle.read() + match = re.search(r'processPath\s*=\s*"([^"]+)"', raw) + args = re.search(r'arguments\s*=\s*"([^"]*)"', raw) + if match: + path = os.path.expandvars(match.group(1)) + report.line('\nhttpPlatform processPath : %s' % match.group(1)) + report.line(' resolved : %s' % path) + report.line(' exists : %s' % os.path.isfile(path)) + if args: + report.line('httpPlatform arguments : %s' % args.group(1)) + except Exception as exc: + report.line('[could not parse web.config: %s]' % exc) + + # ------------------------------------------------------------- 4. app logs + report.head('4. APPLICATION LOGS') + log_dir = os.path.join(APP_ROOT, 'logs') + if not os.path.isdir(log_dir): + report.line('MISSING: %s' % log_dir) + else: + entries = [] + for name in sorted(os.listdir(log_dir)): + full = os.path.join(log_dir, name) + try: + entries.append((os.path.getmtime(full), full, name, + os.path.getsize(full))) + except OSError: + pass + if not entries: + report.line('%s is EMPTY.' % log_dir) + report.line('No stdout log at all means httpPlatformHandler never') + report.line('launched python - look at the pool identity and ACLs.') + for _, full, name, size in sorted(entries, reverse=True)[:5]: + if size == 0: + report.block('%s (0 bytes)' % name, + 'EMPTY - python was launched but wrote nothing.') + continue + try: + with open(full, 'r', encoding='utf-8', errors='replace') as handle: + tail = handle.readlines()[-60:] + report.block('%s (%d bytes, last 60 lines)' % (name, size), + ''.join(tail)) + except Exception as exc: + report.block(name, '[could not read: %s]' % exc) + + # ---------------------------------------------------------------- 5. ACLs + report.head('5. PERMISSIONS') + report.line('The pool runs as "IIS AppPool\\shopdbflask". It needs RX on the') + report.line('tree, Modify on logs and instance, and Read on .env.') + for target in (APP_ROOT, log_dir, os.path.join(APP_ROOT, '.env'), + os.path.join(APP_ROOT, 'venv', 'Scripts')): + if os.path.exists(target): + report.block('icacls %s' % target, run(['icacls', target], timeout=40)) + + # ------------------------------------------------------------ 6. event log + report.head('6. EVENT LOG') + report.block('recent application errors', powershell( + "Get-WinEvent -FilterHashtable @{LogName='Application';" + "StartTime=(Get-Date).AddHours(-6)} -EA SilentlyContinue | " + "Where-Object { $_.ProviderName -match 'HttpPlatform|IIS|W3SVC|WAS|\\.NET' " + "-or $_.LevelDisplayName -eq 'Error' } | Select-Object -First 25 " + "TimeCreated,ProviderName,LevelDisplayName,Message | Format-List | Out-String", + timeout=180)) + report.block('system log: WAS / W3SVC', powershell( + "Get-WinEvent -FilterHashtable @{LogName='System';" + "StartTime=(Get-Date).AddHours(-6)} -EA SilentlyContinue | " + "Where-Object { $_.ProviderName -match 'WAS|W3SVC|HTTP' } | " + "Select-Object -First 20 TimeCreated,ProviderName,LevelDisplayName,Message | " + "Format-List | Out-String", timeout=180)) + + # ------------------------------------------------------------------ write + body = scrub(report.text()) + out_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'shopdb-diagnose-%s.txt' % stamp) + try: + with open(out_path, 'w', encoding='utf-8') as handle: + handle.write(body) + except Exception: + out_path = os.path.join(os.environ.get('TEMP', r'C:\Windows\Temp'), + 'shopdb-diagnose-%s.txt' % stamp) + with open(out_path, 'w', encoding='utf-8') as handle: + handle.write(body) + + print('') + print('Report written to:') + print(' %s' % out_path) + print('') + print('%d secret value(s) were scrubbed from it.' % len(SECRETS)) + print('Send that file back.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/deploy/windows/web.config b/deploy/windows/web.config index 8d0a5c87..f365fcc1 100644 --- a/deploy/windows/web.config +++ b/deploy/windows/web.config @@ -67,15 +67,28 @@ It ships DISABLED because it needs the URL Rewrite module; enabled without it, IIS returns HTTP 500.19 ("configuration section not well-formed"). + There is deliberately NO block below. Setting a + server variable requires that variable to be allowed, but the section + system.webServer/rewrite/allowedServerVariables ships with + overrideModeDefault="Deny", so declaring it in an application's own + web.config is refused outright: IIS answered 500.52 with error 0x80070021, + "this configuration section cannot be used at this path", BEFORE it ever + reached httpPlatformHandler - so python was never launched and the stdout + log stayed empty, which looks like an application fault and is not one. + + The installer instead allows the single variable at server level, which + grants exactly HTTP_X_FORWARDED_FOR rather than unlocking the section and + letting every site on the machine declare arbitrary server variables. + The installer handles both: -ClientIpSource direct installs URL Rewrite - from the bundle and enables this; -ClientIpSource proxy leaves it alone. - By hand: install URL Rewrite, then delete the two marker lines below. + from the bundle, allows the variable, and enables this; -ClientIpSource + proxy leaves it alone. By hand: install URL Rewrite, run + appcmd set config /section:system.webServer/rewrite/allowedServerVariables ^ + /+"[name='HTTP_X_FORWARDED_FOR']" /commit:apphost + then delete the two marker lines below. -->