distribute prebuilt channel images via GHCR instead of building on pilots

Pilots behind the inspecting proxy cannot reliably build the frontend from source; npm's many small fetches get reset mid-install. Build the image once in CI on the clean runners and have pilots pull the finished channel image. One resumable image pull survives the proxy where hundreds of npm requests do not, and a pilot now needs only Docker.
2026-09-02 09:34:40 -04:00
parent a299e751dc
commit d0e9f80287
4 changed files with 224 additions and 25 deletions
--- a/.github/workflows/publish-images.yml
+++ b/.github/workflows/publish-images.yml
@@ -0,0 +1,81 @@
+# Build and publish per-channel container images to GHCR.
+#
+# WHY this exists: pilot dev workstations sit behind an inspecting proxy
+# (Zscaler) that resets npm's many small fetches mid-install, so building the
+# image FROM SOURCE on a pilot box is unreliable. These runners
+# (arc-runner-set) install npm cleanly - the CI `frontend` job already proves
+# it - so we build the image ONCE here and pilots pull the finished image.
+# A single image pull is one resumable HTTPS stream, far more robust through
+# the proxy than hundreds of npm requests.
+#
+# Channels map to branches:
+#   main   -> ghcr.io/<owner>/shopdb-flask:stable  (badge off)
+#   dev-be -> ghcr.io/<owner>/shopdb-flask:edge    (SHOPDB_ENV_LABEL=DEV badge)
+#
+# Each push to those branches rebuilds and republishes its channel tag, so
+# `edge` stays genuinely bleeding edge. The image is also tagged with the short
+# commit SHA for traceability and rollback.
+
+name: publish-images
+
+on:
+  push:
+    branches:
+      - main
+      - dev-be
+  # Allow a manual rebuild/publish of the current branch's channel image.
+  workflow_dispatch:
+
+# One in-flight publish per branch; a newer push supersedes an older one.
+concurrency:
+  group: publish-images-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-push:
+    runs-on: arc-runner-set
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v5
+
+      # Lowercase the owner: GHCR repository paths must be lowercase, but
+      # github.repository_owner can carry uppercase (e.g. ge-aero is fine, but
+      # this keeps the workflow correct for any owner casing).
+      - name: Resolve image name and channel tag
+        id: meta
+        run: |
+          owner="$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')"
+          image="ghcr.io/${owner}/shopdb-flask"
+          if [ "${{ github.ref_name }}" = "main" ]; then
+            channel="stable"
+          else
+            channel="edge"
+          fi
+          echo "image=${image}" >> "$GITHUB_OUTPUT"
+          echo "channel=${channel}" >> "$GITHUB_OUTPUT"
+          echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push channel image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.channel }}
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.channel }}-${{ steps.meta.outputs.sha }}
+          # Reuse the channel tag as a cache source/target so subsequent builds
+          # only redo layers that actually changed.
+          cache-from: type=registry,ref=${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.channel }}
+          cache-to: type=inline
--- a/docker-compose.pull.yml
+++ b/docker-compose.pull.yml
@@ -0,0 +1,92 @@
+# shopdb-flask PULL stack for dev workstations (stable / edge channels).
+#
+# WHY this exists: pilots behind an inspecting proxy cannot reliably BUILD the
+# image from source (npm's many small fetches get reset mid-install). Instead
+# CI (.github/workflows/publish-images.yml) builds the image on clean runners
+# and publishes it to GHCR; this file just PULLS and RUNS it. One resumable
+# image pull survives the proxy where hundreds of npm requests do not.
+#
+# Nothing is built here (no `build:`), mirroring docker-compose.airgap.yml:
+#   - api/migrate use `image:` (pulled from GHCR), never `build: .`.
+#   - NO ./plugins bind mount; the image carries every plugin baked in.
+#   - a one-shot `migrate` runs db upgrade + plugin upgrade-all + seed before
+#     api starts, so `up -d` alone brings up a working site.
+#
+# Driven by scripts/dev-bootstrap.ps1, which sets SHOPDB_IMAGE (the channel
+# image ref), SHOPDB_ENV_LABEL, API_PORT and MYSQL_PORT per channel and picks a
+# compose project name (-p shopdb-stable / -p shopdb-edge) so the two channels
+# run side by side without colliding.
+
+x-app-env: &app-env
+  FLASK_APP: wsgi.py
+  FLASK_ENV: production
+  DATABASE_URL: mysql+pymysql://shopdb:${MYSQL_PASSWORD}@db:3306/shopdb_flask?charset=utf8mb4
+  SECRET_KEY: ${SECRET_KEY:?SECRET_KEY must be set}
+  JWT_SECRET_KEY: ${JWT_SECRET_KEY:?JWT_SECRET_KEY must be set}
+  CORS_ORIGINS: ${CORS_ORIGINS:?CORS_ORIGINS must be set}
+  LOG_LEVEL: ${LOG_LEVEL:-INFO}
+  # Empty on stable; 'DEV' on edge, which lights the sidebar badge.
+  SHOPDB_ENV_LABEL: ${SHOPDB_ENV_LABEL:-}
+
+services:
+  db:
+    image: mysql:8.4
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    restart: unless-stopped
+    environment:
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?MYSQL_ROOT_PASSWORD must be set}
+      MYSQL_DATABASE: shopdb_flask
+      MYSQL_USER: shopdb
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD:?MYSQL_PASSWORD must be set}
+    volumes:
+      - db_data:/var/lib/mysql
+    ports:
+      - "127.0.0.1:${MYSQL_PORT:-3306}:3306"
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-u", "root", "-p${MYSQL_ROOT_PASSWORD}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # One-shot schema + seed. Runs to completion and exits; api waits for it.
+  # Every step is idempotent, so it is safe to run on every `up`.
+  migrate:
+    image: ${SHOPDB_IMAGE:?SHOPDB_IMAGE must be set to the channel image ref}
+    pull_policy: always
+    restart: "no"
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      <<: *app-env
+    command:
+      - sh
+      - -c
+      - >
+        flask db upgrade &&
+        flask plugin upgrade-all &&
+        flask seed permissions &&
+        flask seed settings &&
+        flask seed reference-data
+    volumes:
+      - instance_data:/app/instance
+
+  api:
+    image: ${SHOPDB_IMAGE:?SHOPDB_IMAGE must be set to the channel image ref}
+    pull_policy: always
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+      migrate:
+        condition: service_completed_successfully
+    environment:
+      <<: *app-env
+    ports:
+      - "${API_PORT:-5001}:5001"
+    volumes:
+      - instance_data:/app/instance
+
+volumes:
+  db_data:
+  instance_data:
--- a/docs/DEV-BUILD-TEST.md
+++ b/docs/DEV-BUILD-TEST.md
@@ -1,19 +1,21 @@
 # ShopDB dev build - quick test

 Two dev builds you can stand up on a Windows box with Docker Desktop. You run
-one PowerShell script: it clones the repo, checks your dependencies (offers to
-winget-install anything missing), then builds and starts the stack in Docker.
+one PowerShell script: it checks your dependencies (offers to winget-install
+anything missing), then PULLS a prebuilt image from GHCR and starts the stack in
+Docker. Nothing is compiled on your machine - no Node, no npm, no source build -
+so it works on a box that has only Docker installed.

 ## What you need

 - Windows with Docker Desktop running.
 - Git for Windows (`winget install Git.Git`).
- A GitHub login for this private repo already stored on the box. If you have
-  cloned this repo before, or signed in to GitHub in Visual Studio or VS Code,
-  you already have it - the script uses that stored credential automatically, so
-  there is no token to paste. If you have never signed in, do one manual
-  `git clone https://github.com/ge-aero/shopdb-flask.git` first (it will prompt
-  you to sign in), then delete that clone - it is only to seed the credential.
+- A GitHub login for this private repo/registry already stored on the box. If
+  you have cloned this repo before, or signed in to GitHub in Visual Studio or
+  VS Code, you already have it - the script uses that stored credential
+  automatically. If Docker cannot pull the image, sign in once with
+  `docker login ghcr.io` (use a GitHub username and a personal access token that
+  has `read:packages`), then re-run the launcher.

 ## Get the launcher

@@ -24,18 +26,20 @@ your Downloads folder) and open PowerShell in that folder.

 	powershell -ExecutionPolicy Bypass -File .\Start-ShopDBDev.ps1 -Channel edge

-Comes up at https://localhost:5011. Red DEV badge in the sidebar.
+Comes up at https://localhost:5011. Red DEV badge in the sidebar. Pulls the
+`edge` image.

 ## Stable (main)

 	powershell -ExecutionPolicy Bypass -File .\Start-ShopDBDev.ps1 -Channel stable

-Comes up at https://localhost:5001. No DEV badge.
+Comes up at https://localhost:5001. No DEV badge. Pulls the `stable` image.

 ## Notes

- Both builds can run at the same time - they use different ports and separate
+- Both stacks can run at the same time - they use different ports and separate
  Docker volumes, so they do not collide and each keeps its own database.
- The code is cloned to `%USERPROFILE%\shopdb-dev` (override with `-TargetDir`).
+- Each launch pulls the latest image for the channel, so re-running the launcher
+  is how you pick up a new build. No local rebuild step.
 - First page load shows a self-signed cert warning (expected for a dev box).
 - Stop a stack:  `docker compose -p shopdb-stable down`  (or `shopdb-edge`).
--- a/scripts/dev-bootstrap.ps1
+++ b/scripts/dev-bootstrap.ps1
@@ -11,12 +11,15 @@
 		 silently). Python is optional and only checked when -CheckPython is
 		 passed, because the app itself runs entirely in the container.
 	  2. Picks a channel:
-		   stable  -> builds from the 'main' branch, no DEV badge.
-		   edge    -> builds from the 'dev-be' branch, red DEV badge in the UI.
+		   stable  -> pulls the 'stable' image (built from main), no DEV badge.
+		   edge    -> pulls the 'edge' image (built from dev-be), red DEV badge.
 		 The two channels use different compose project names, host ports, and
 		 named volumes, so they run side by side without colliding.
 	  3. Generates a .env for the channel (dev-grade secrets, NOT for production)
-		 if one does not already exist, then builds and brings up the stack.
+		 if one does not already exist, then PULLS the prebuilt channel image
+		 from GHCR and brings up the stack. Nothing is built on the host: the
+		 image is produced by CI, so a pilot needs only Docker (no Node/npm and
+		 no source compile behind the corporate proxy).

 	Windows-only by design: pre-release audience is Windows. Not a production
 	installer - see deploy/windows/installer/shopdb-install.ps1 for that.
@@ -26,7 +29,8 @@

 .PARAMETER Ref
 	Override the git ref to build. Defaults to 'main' for stable, 'dev-be' for
-	edge. Useful for testing a feature branch through the edge plumbing.
+	edge. Retained for compatibility; the pull flow selects the image by channel
+	tag, so this now only affects which branch the working tree is prepared on.

 .PARAMETER CheckPython
 	Also verify a local Python 3.11+ (only needed if you run tests/tooling on
@@ -67,6 +71,7 @@ $channels = @{
 		ApiPort   = 5001
 		MysqlPort = 3306
 		EnvLabel  = ''
+		ImageTag  = 'stable'
 	}
 	edge = @{
 		Ref       = 'dev-be'
@@ -74,9 +79,15 @@ $channels = @{
 		ApiPort   = 5011
 		MysqlPort = 3316
 		EnvLabel  = 'DEV'
+		ImageTag  = 'edge'
 	}
 }

+# GHCR repository that CI (.github/workflows/publish-images.yml) publishes the
+# per-channel images to. Pilots PULL this instead of building from source,
+# because building behind an inspecting proxy is unreliable.
+$imageRepo = 'ghcr.io/ge-aero/shopdb-flask'
+
 function Write-Step {
 	param([string]$Message)
 	Write-Host ''
@@ -259,6 +270,15 @@ try {
 	$envPath = Join-Path $repoRoot ".env.$($config.Project)"
 	if (Test-Path $envPath) {
 		Write-Ok "Reusing existing $([System.IO.Path]::GetFileName($envPath))."
+		# A .env written before the pull-based flow existed will not carry a
+		# SHOPDB_IMAGE line, and the pull compose file requires it. Top it up in
+		# place rather than force the pilot to delete and re-key the file.
+		$image = "$imageRepo`:$($config.ImageTag)"
+		$envText = Get-Content -Path $envPath
+		if (-not ($envText | Where-Object { $_ -match '^SHOPDB_IMAGE=' })) {
+			Add-Content -Path $envPath -Value "SHOPDB_IMAGE=$image" -Encoding ASCII
+			Write-Ok "Added SHOPDB_IMAGE=$image to the existing .env."
+		}
 	} else {
 		# Dev-grade secrets: generated locally, never for production. A real
 		# deploy fills .env from the wiki runbook, not from this script.
@@ -268,6 +288,7 @@ try {
 		$rootPass = [Convert]::ToBase64String((1..18 | ForEach-Object { Get-Random -Maximum 256 })) -replace '[^A-Za-z0-9]', 'x'
 		$apiPort = $config.ApiPort
 		$mysqlPort = $config.MysqlPort
+		$image = "$imageRepo`:$($config.ImageTag)"

 		$lines = @(
 			"# Auto-generated by scripts/dev-bootstrap.ps1 for the $Channel channel.",
@@ -283,28 +304,29 @@ try {
 			"MYSQL_PASSWORD=$dbPass",
 			"MYSQL_PORT=$mysqlPort",
 			"API_PORT=$apiPort",
+			"SHOPDB_IMAGE=$image",
 			"SHOPDB_ENV_LABEL=$($config.EnvLabel)"
 		)
 		Set-Content -Path $envPath -Value $lines -Encoding ASCII
 		Write-Ok "Wrote $([System.IO.Path]::GetFileName($envPath)) (API on $apiPort, MySQL on $mysqlPort)."
 	}

-	# --- 4. Build and start ---------------------------------------------------
+	# --- 4. Pull and start ----------------------------------------------------

-	Write-Step 'Detecting corporate TLS-inspection CA for the Docker build'
-	Set-CorpRootCa -RepoRoot $repoRoot
-
-	Write-Step "Building and starting the '$($config.Project)' stack"
+	# No local build: the image is produced by CI and published to GHCR (see
+	# .github/workflows/publish-images.yml). Pilots may have only Docker
+	# installed, and building the frontend from source behind an inspecting
+	# proxy is unreliable, so we pull the finished channel image instead.
+	Write-Step "Pulling and starting the '$($config.Project)' stack"
 	$composeArgs = @(
 		'compose',
 		'-p', $config.Project,
 		'--env-file', $envPath,
-		'-f', 'docker-compose.yml',
-		'-f', 'docker-compose.channel.yml'
+		'-f', 'docker-compose.pull.yml'
 	)

-	& docker @composeArgs build api
-	if ($LASTEXITCODE -ne 0) { throw "docker compose build failed ($LASTEXITCODE)." }
+	& docker @composeArgs pull
+	if ($LASTEXITCODE -ne 0) { throw "docker compose pull failed ($LASTEXITCODE). If this is an auth error, run: docker login ghcr.io" }

 	& docker @composeArgs up -d
 	if ($LASTEXITCODE -ne 0) { throw "docker compose up failed ($LASTEXITCODE)." }