diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml new file mode 100644 index 00000000..16bea88a --- /dev/null +++ b/.github/workflows/publish-images.yml @@ -0,0 +1,81 @@ +# Build and publish per-channel container images to GHCR. +# +# WHY this exists: pilot dev workstations sit behind an inspecting proxy +# (Zscaler) that resets npm's many small fetches mid-install, so building the +# image FROM SOURCE on a pilot box is unreliable. These runners +# (arc-runner-set) install npm cleanly - the CI `frontend` job already proves +# it - so we build the image ONCE here and pilots pull the finished image. +# A single image pull is one resumable HTTPS stream, far more robust through +# the proxy than hundreds of npm requests. +# +# Channels map to branches: +# main -> ghcr.io//shopdb-flask:stable (badge off) +# dev-be -> ghcr.io//shopdb-flask:edge (SHOPDB_ENV_LABEL=DEV badge) +# +# Each push to those branches rebuilds and republishes its channel tag, so +# `edge` stays genuinely bleeding edge. The image is also tagged with the short +# commit SHA for traceability and rollback. + +name: publish-images + +on: + push: + branches: + - main + - dev-be + # Allow a manual rebuild/publish of the current branch's channel image. + workflow_dispatch: + +# One in-flight publish per branch; a newer push supersedes an older one. +concurrency: + group: publish-images-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: arc-runner-set + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v5 + + # Lowercase the owner: GHCR repository paths must be lowercase, but + # github.repository_owner can carry uppercase (e.g. ge-aero is fine, but + # this keeps the workflow correct for any owner casing). + - name: Resolve image name and channel tag + id: meta + run: | + owner="$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" + image="ghcr.io/${owner}/shopdb-flask" + if [ "${{ github.ref_name }}" = "main" ]; then + channel="stable" + else + channel="edge" + fi + echo "image=${image}" >> "$GITHUB_OUTPUT" + echo "channel=${channel}" >> "$GITHUB_OUTPUT" + echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push channel image + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.channel }} + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.channel }}-${{ steps.meta.outputs.sha }} + # Reuse the channel tag as a cache source/target so subsequent builds + # only redo layers that actually changed. + cache-from: type=registry,ref=${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.channel }} + cache-to: type=inline diff --git a/docker-compose.pull.yml b/docker-compose.pull.yml new file mode 100644 index 00000000..1d04ec1b --- /dev/null +++ b/docker-compose.pull.yml @@ -0,0 +1,92 @@ +# shopdb-flask PULL stack for dev workstations (stable / edge channels). +# +# WHY this exists: pilots behind an inspecting proxy cannot reliably BUILD the +# image from source (npm's many small fetches get reset mid-install). Instead +# CI (.github/workflows/publish-images.yml) builds the image on clean runners +# and publishes it to GHCR; this file just PULLS and RUNS it. One resumable +# image pull survives the proxy where hundreds of npm requests do not. +# +# Nothing is built here (no `build:`), mirroring docker-compose.airgap.yml: +# - api/migrate use `image:` (pulled from GHCR), never `build: .`. +# - NO ./plugins bind mount; the image carries every plugin baked in. +# - a one-shot `migrate` runs db upgrade + plugin upgrade-all + seed before +# api starts, so `up -d` alone brings up a working site. +# +# Driven by scripts/dev-bootstrap.ps1, which sets SHOPDB_IMAGE (the channel +# image ref), SHOPDB_ENV_LABEL, API_PORT and MYSQL_PORT per channel and picks a +# compose project name (-p shopdb-stable / -p shopdb-edge) so the two channels +# run side by side without colliding. + +x-app-env: &app-env + FLASK_APP: wsgi.py + FLASK_ENV: production + DATABASE_URL: mysql+pymysql://shopdb:${MYSQL_PASSWORD}@db:3306/shopdb_flask?charset=utf8mb4 + SECRET_KEY: ${SECRET_KEY:?SECRET_KEY must be set} + JWT_SECRET_KEY: ${JWT_SECRET_KEY:?JWT_SECRET_KEY must be set} + CORS_ORIGINS: ${CORS_ORIGINS:?CORS_ORIGINS must be set} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + # Empty on stable; 'DEV' on edge, which lights the sidebar badge. + SHOPDB_ENV_LABEL: ${SHOPDB_ENV_LABEL:-} + +services: + db: + image: mysql:8.4 + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?MYSQL_ROOT_PASSWORD must be set} + MYSQL_DATABASE: shopdb_flask + MYSQL_USER: shopdb + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?MYSQL_PASSWORD must be set} + volumes: + - db_data:/var/lib/mysql + ports: + - "127.0.0.1:${MYSQL_PORT:-3306}:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-u", "root", "-p${MYSQL_ROOT_PASSWORD}"] + interval: 10s + timeout: 5s + retries: 5 + + # One-shot schema + seed. Runs to completion and exits; api waits for it. + # Every step is idempotent, so it is safe to run on every `up`. + migrate: + image: ${SHOPDB_IMAGE:?SHOPDB_IMAGE must be set to the channel image ref} + pull_policy: always + restart: "no" + depends_on: + db: + condition: service_healthy + environment: + <<: *app-env + command: + - sh + - -c + - > + flask db upgrade && + flask plugin upgrade-all && + flask seed permissions && + flask seed settings && + flask seed reference-data + volumes: + - instance_data:/app/instance + + api: + image: ${SHOPDB_IMAGE:?SHOPDB_IMAGE must be set to the channel image ref} + pull_policy: always + restart: unless-stopped + depends_on: + db: + condition: service_healthy + migrate: + condition: service_completed_successfully + environment: + <<: *app-env + ports: + - "${API_PORT:-5001}:5001" + volumes: + - instance_data:/app/instance + +volumes: + db_data: + instance_data: diff --git a/docs/DEV-BUILD-TEST.md b/docs/DEV-BUILD-TEST.md index 4db2abd7..ac357d1b 100644 --- a/docs/DEV-BUILD-TEST.md +++ b/docs/DEV-BUILD-TEST.md @@ -1,19 +1,21 @@ # ShopDB dev build - quick test Two dev builds you can stand up on a Windows box with Docker Desktop. You run -one PowerShell script: it clones the repo, checks your dependencies (offers to -winget-install anything missing), then builds and starts the stack in Docker. +one PowerShell script: it checks your dependencies (offers to winget-install +anything missing), then PULLS a prebuilt image from GHCR and starts the stack in +Docker. Nothing is compiled on your machine - no Node, no npm, no source build - +so it works on a box that has only Docker installed. ## What you need - Windows with Docker Desktop running. - Git for Windows (`winget install Git.Git`). -- A GitHub login for this private repo already stored on the box. If you have - cloned this repo before, or signed in to GitHub in Visual Studio or VS Code, - you already have it - the script uses that stored credential automatically, so - there is no token to paste. If you have never signed in, do one manual - `git clone https://github.com/ge-aero/shopdb-flask.git` first (it will prompt - you to sign in), then delete that clone - it is only to seed the credential. +- A GitHub login for this private repo/registry already stored on the box. If + you have cloned this repo before, or signed in to GitHub in Visual Studio or + VS Code, you already have it - the script uses that stored credential + automatically. If Docker cannot pull the image, sign in once with + `docker login ghcr.io` (use a GitHub username and a personal access token that + has `read:packages`), then re-run the launcher. ## Get the launcher @@ -24,18 +26,20 @@ your Downloads folder) and open PowerShell in that folder. powershell -ExecutionPolicy Bypass -File .\Start-ShopDBDev.ps1 -Channel edge -Comes up at https://localhost:5011. Red DEV badge in the sidebar. +Comes up at https://localhost:5011. Red DEV badge in the sidebar. Pulls the +`edge` image. ## Stable (main) powershell -ExecutionPolicy Bypass -File .\Start-ShopDBDev.ps1 -Channel stable -Comes up at https://localhost:5001. No DEV badge. +Comes up at https://localhost:5001. No DEV badge. Pulls the `stable` image. ## Notes -- Both builds can run at the same time - they use different ports and separate +- Both stacks can run at the same time - they use different ports and separate Docker volumes, so they do not collide and each keeps its own database. -- The code is cloned to `%USERPROFILE%\shopdb-dev` (override with `-TargetDir`). +- Each launch pulls the latest image for the channel, so re-running the launcher + is how you pick up a new build. No local rebuild step. - First page load shows a self-signed cert warning (expected for a dev box). - Stop a stack: `docker compose -p shopdb-stable down` (or `shopdb-edge`). diff --git a/scripts/dev-bootstrap.ps1 b/scripts/dev-bootstrap.ps1 index 20b9fefa..6a95a098 100644 --- a/scripts/dev-bootstrap.ps1 +++ b/scripts/dev-bootstrap.ps1 @@ -11,12 +11,15 @@ silently). Python is optional and only checked when -CheckPython is passed, because the app itself runs entirely in the container. 2. Picks a channel: - stable -> builds from the 'main' branch, no DEV badge. - edge -> builds from the 'dev-be' branch, red DEV badge in the UI. + stable -> pulls the 'stable' image (built from main), no DEV badge. + edge -> pulls the 'edge' image (built from dev-be), red DEV badge. The two channels use different compose project names, host ports, and named volumes, so they run side by side without colliding. 3. Generates a .env for the channel (dev-grade secrets, NOT for production) - if one does not already exist, then builds and brings up the stack. + if one does not already exist, then PULLS the prebuilt channel image + from GHCR and brings up the stack. Nothing is built on the host: the + image is produced by CI, so a pilot needs only Docker (no Node/npm and + no source compile behind the corporate proxy). Windows-only by design: pre-release audience is Windows. Not a production installer - see deploy/windows/installer/shopdb-install.ps1 for that. @@ -26,7 +29,8 @@ .PARAMETER Ref Override the git ref to build. Defaults to 'main' for stable, 'dev-be' for - edge. Useful for testing a feature branch through the edge plumbing. + edge. Retained for compatibility; the pull flow selects the image by channel + tag, so this now only affects which branch the working tree is prepared on. .PARAMETER CheckPython Also verify a local Python 3.11+ (only needed if you run tests/tooling on @@ -67,6 +71,7 @@ $channels = @{ ApiPort = 5001 MysqlPort = 3306 EnvLabel = '' + ImageTag = 'stable' } edge = @{ Ref = 'dev-be' @@ -74,9 +79,15 @@ $channels = @{ ApiPort = 5011 MysqlPort = 3316 EnvLabel = 'DEV' + ImageTag = 'edge' } } +# GHCR repository that CI (.github/workflows/publish-images.yml) publishes the +# per-channel images to. Pilots PULL this instead of building from source, +# because building behind an inspecting proxy is unreliable. +$imageRepo = 'ghcr.io/ge-aero/shopdb-flask' + function Write-Step { param([string]$Message) Write-Host '' @@ -259,6 +270,15 @@ try { $envPath = Join-Path $repoRoot ".env.$($config.Project)" if (Test-Path $envPath) { Write-Ok "Reusing existing $([System.IO.Path]::GetFileName($envPath))." + # A .env written before the pull-based flow existed will not carry a + # SHOPDB_IMAGE line, and the pull compose file requires it. Top it up in + # place rather than force the pilot to delete and re-key the file. + $image = "$imageRepo`:$($config.ImageTag)" + $envText = Get-Content -Path $envPath + if (-not ($envText | Where-Object { $_ -match '^SHOPDB_IMAGE=' })) { + Add-Content -Path $envPath -Value "SHOPDB_IMAGE=$image" -Encoding ASCII + Write-Ok "Added SHOPDB_IMAGE=$image to the existing .env." + } } else { # Dev-grade secrets: generated locally, never for production. A real # deploy fills .env from the wiki runbook, not from this script. @@ -268,6 +288,7 @@ try { $rootPass = [Convert]::ToBase64String((1..18 | ForEach-Object { Get-Random -Maximum 256 })) -replace '[^A-Za-z0-9]', 'x' $apiPort = $config.ApiPort $mysqlPort = $config.MysqlPort + $image = "$imageRepo`:$($config.ImageTag)" $lines = @( "# Auto-generated by scripts/dev-bootstrap.ps1 for the $Channel channel.", @@ -283,28 +304,29 @@ try { "MYSQL_PASSWORD=$dbPass", "MYSQL_PORT=$mysqlPort", "API_PORT=$apiPort", + "SHOPDB_IMAGE=$image", "SHOPDB_ENV_LABEL=$($config.EnvLabel)" ) Set-Content -Path $envPath -Value $lines -Encoding ASCII Write-Ok "Wrote $([System.IO.Path]::GetFileName($envPath)) (API on $apiPort, MySQL on $mysqlPort)." } - # --- 4. Build and start --------------------------------------------------- + # --- 4. Pull and start ---------------------------------------------------- - Write-Step 'Detecting corporate TLS-inspection CA for the Docker build' - Set-CorpRootCa -RepoRoot $repoRoot - - Write-Step "Building and starting the '$($config.Project)' stack" + # No local build: the image is produced by CI and published to GHCR (see + # .github/workflows/publish-images.yml). Pilots may have only Docker + # installed, and building the frontend from source behind an inspecting + # proxy is unreliable, so we pull the finished channel image instead. + Write-Step "Pulling and starting the '$($config.Project)' stack" $composeArgs = @( 'compose', '-p', $config.Project, '--env-file', $envPath, - '-f', 'docker-compose.yml', - '-f', 'docker-compose.channel.yml' + '-f', 'docker-compose.pull.yml' ) - & docker @composeArgs build api - if ($LASTEXITCODE -ne 0) { throw "docker compose build failed ($LASTEXITCODE)." } + & docker @composeArgs pull + if ($LASTEXITCODE -ne 0) { throw "docker compose pull failed ($LASTEXITCODE). If this is an auth error, run: docker login ghcr.io" } & docker @composeArgs up -d if ($LASTEXITCODE -ne 0) { throw "docker compose up failed ($LASTEXITCODE)." }