shopdb-flask/tools/export-github.sh

#!/bin/bash
# Export the working repo to the GitHub publication repo and emit a bundle.
#
# Pipeline: working repo (full history, internal refs) -> pruned/scrubbed
# tree -> commit in ~/projects/shopdb-flask-pub (the local mirror of what
# enterprise GitHub holds) -> full git bundle in /home/camp/pxe-images/ for
# transfer to the work PC, which pushes it to GitHub.
#
# Usage:
#   tools/export-github.sh "Commit message for the publication commit"
#   tools/export-github.sh --dist          # also rebuild BOTH frontend dists
#                                          # (/ops = dev, /shopdb = prod)
#
# This script lives in tools/, which is itself excluded from publication.
set -euo pipefail

WORK=/home/camp/projects/shopdb-flask
PUB=/home/camp/projects/shopdb-flask-pub
OUT=/home/camp/pxe-images/github

BUILD_DIST=0
MSG=""
for arg in "$@"; do
  case "$arg" in
    --dist) BUILD_DIST=1 ;;
    *) MSG="$arg" ;;
  esac
done
[ -n "$MSG" ] || { echo "usage: $0 [--dist] \"commit message\""; exit 1; }
[ -d "$PUB/.git" ] || { echo "publication repo missing at $PUB"; exit 1; }

# --- 0. which branch is being published -------------------------------------
#
# This syncs the WORKING TREE, not a branch, and it used to commit whatever it
# found onto the publication repo's main and print "git push origin main". So
# running it while checked out on a site branch published that site's work AS
# THE PRODUCT, to the repository every other site consumes, without saying a
# word about it.
#
# The publication repo now mirrors the branch you are actually on.
BRANCH="$(git -C "$WORK" rev-parse --abbrev-ref HEAD)"
if [ "$BRANCH" = "HEAD" ]; then
  echo "detached HEAD in $WORK - check out a branch before exporting" >&2
  exit 1
fi
echo "==> publishing branch: $BRANCH"

# --- 1. sync the tree (working -> pub), minus everything never published ---
#
# NOTE the deploy/windows/installer/bundle exclude below. That is ~100MB of build
# output holding a COPY of the whole application tree plus the wheels and vendor
# installers. It is gitignored, but this sync walks the WORKING TREE rather than
# git, so it came through anyway - and its copies of config.py and
# requirements.txt then tripped the scrub gate. Anything else generated into the
# working tree needs excluding here too, for the same reason.
# rsync --exclude also PROTECTS a path from --delete, so anything that reached
# the publication tree before its exclude existed stays there forever, invisible
# to the sync and caught only by the scrub gate. Purge the generated paths first
# so adding an exclude is enough on its own.
#
# docs/ does NOT publish - the prose becomes the WIKI instead. One exception,
# and it is not a documentation decision:
#
#   docs/openapi.json is read AT RUNTIME by shopdb/core/api/docs.py, which
#   serves the /api/docs page every published site ships. Excluding it meant
#   that page loaded and then answered {"error":"openapi.json not generated"}
#   on every site installed from the published repo. It is a product file that
#   happens to live under docs/, not documentation.
#
# It also un-skips tests/test_openapi_spec.py, which had been reporting green
# by not running - see scripts/check-gates-ran.sh.
#
# docs/api-inventory.json stays OUT: nothing reads it at runtime, so it is a
# development gate and belongs with the wiki. That does mean
# test_api_inventory_parity cannot run on GitHub, which is why the gate guard
# separates the checks that must run everywhere from the ones that only run
# where the repo is complete.
# instance/plugins.json is one site's plugin CHOICES. It is excluded, so it was
# never published - but an old copy sat in the publication tree for months,
# protected from --delete by that same exclude, and it made the published repo's
# own test suite fail locally: 13 plugins enabled where the code expects 16.
# A publication tree that cannot pass its own tests is not one anybody can
# verify before shipping.
for stale in .pytest_cache .ruff_cache htmlcov .coverage instance/plugins.json \
             deploy/windows/installer/bundle deploy/windows/installer/Output \
             deploy/windows/installer/plugins.iss deploy/windows/installer/version.iss; do
  rm -rf "${PUB:?}/$stale"
done

# .env is excluded by name, but a stray copy BESIDE it is not: a hand-made
# .env.bak-<date> was picked up by this sync, because it walks the WORKING TREE
# and .gitignore does not apply to it. It got no further than the scrub gate,
# which is what that gate is for, but a file holding live credentials should
# never reach the publication tree at all. Hence --exclude '.env.*', with
# --include '.env.example' BEFORE it: rsync takes the first matching rule, and
# .env.example does publish.
#
# Keep flags on their own lines. A comment inside this backslash-continued
# command is not a comment - bash hands the text to rsync as arguments, and it
# dies with a usage dump rather than anything that names the real problem.
# Mirror the branch in the publication repo BEFORE the sync.
#
# Order matters and got this wrong once: done after the rsync, `git checkout
# --force` reverts the tree to the branch's committed state and throws away
# everything rsync just wrote, so the export reports "no changes" and publishes
# nothing. It is a silent no-op, which is the worst shape for this failure -
# the script still prints a bundle path and looks like it worked.
#
# --force is needed because the tree may hold leftovers from the previously
# published branch, and safe because this tree is a mirror: the rsync below
# replaces it wholesale from $WORK.
(
  cd "$PUB"
  if git show-ref --verify --quiet "refs/heads/$BRANCH"; then
    git checkout --force "$BRANCH" >/dev/null
  else
    # A new site branch starts from main, as it does in the dev repo, so its
    # history is the product's plus whatever it adds.
    echo "==> publication repo: creating branch $BRANCH from main"
    git checkout --force -b "$BRANCH" main >/dev/null
  fi
)

rsync -a --delete \
  --exclude '.git' \
  --exclude '.gitea' \
  --include '/docs/' \
  --include '/docs/openapi.json' \
  --exclude '/docs/*' \
  --exclude '/tools' \
  --exclude '/mcp' \
  --exclude 'start-api.sh' \
  --exclude 'start-ui.sh' \
  --exclude 'CLAUDE.md' \
  --exclude 'frontend/CLAUDE.md' \
  --exclude 'tests/test_docs_contract.py' \
  --exclude 'tests/test_plugins/test_geenforce_parity.py' \
  --exclude 'tests/test_plugins/test_zabbix_live.py' \
  --exclude 'venv' \
  --exclude 'node_modules' \
  --exclude 'deploy/windows/installer/bundle' \
  --exclude 'deploy/windows/installer/Output' \
  --exclude 'deploy/windows/installer/plugins.iss' \
  --exclude 'deploy/windows/installer/version.iss' \
  --exclude 'frontend/dist*' \
  --exclude 'frontend/src/.plugins-staged' \
  --exclude 'frontend/src/router/routes.gen.js' \
  --exclude '/instance' \
  --include '.env.example' \
  --exclude '.env.*' \
  --exclude '.env' \
  --exclude '__pycache__' \
  --exclude '.pytest_cache' \
  --exclude '.ruff_cache' \
  --exclude '.coverage' \
  --exclude 'htmlcov' \
  --exclude '*.pyc' \
  --exclude 'scripts/site_imports/wjf/idmap.json' \
  "$WORK/" "$PUB/"

# --- 1b. every bundled plugin must have survived the sync ---
#
# The excludes above are rsync patterns, and a pattern with no leading slash
# matches at ANY depth. 'tools' was meant to drop the repo-root tools/ dir (this
# script lives in it) and silently dropped plugins/tools/ as well, so a plugin
# was committed, exported, bundled and deployed without ever being in the
# payload - the failure only surfaced as a missing manifest.json on the server.
# The excludes are anchored now; this check is what makes a recurrence loud.
missing=""
for manifest in "$WORK"/plugins/*/manifest.json; do
  [ -e "$manifest" ] || continue
  plugin=$(basename "$(dirname "$manifest")")
  [ -f "$PUB/plugins/$plugin/manifest.json" ] || missing="$missing $plugin"
done
if [ -n "$missing" ]; then
  echo "ERROR: bundled plugin(s) missing from the publication tree:$missing" >&2
  echo "       an rsync --exclude above is matching them; anchor it with a leading slash." >&2
  exit 1
fi

cd "$PUB"

# --- 2. re-apply the publication wording (idempotent) ---
# docs/ lives only in the wiki on the GitHub side.
# NOTE the `|| true` on each scrub grep below. grep exits 1 when it matches
# NOTHING, and under `set -euo pipefail` that failing pipeline aborts the whole
# script - silently, because grep prints nothing to say so. That is exactly what
# happened once every internal reference had been scrubbed: the gitea grep found
# no matches, the export died before committing or rebuilding the dists, and it
# exited without a single line of output. A scrub that finds nothing left to
# scrub is the SUCCESS case, so it must not be an error.
{ grep -rlZ 'docs/' --include='*.py' --include='*.md' --include='*.sh' \
    --include='*.tmpl' --include='*.vue' --include='*.example' . 2>/dev/null || true; } |
  while IFS= read -r -d '' f; do
    case "$f" in ./CHANGELOG.md) continue ;; esac
    sed -i -E \
      -e 's/\[`?docs\/([A-Za-z0-9_-]+)\.md`?\]\((\.\.\/)*docs\/[A-Za-z0-9_-]+\.md\)/the \1 page in the project wiki/g' \
      -e 's/docs\/proposals\/ge-enforce-plugin\.md/the ge-enforce-plugin proposal in the project wiki/g' \
      -e 's/`docs\/([A-Za-z0-9_-]+)\.md`/the \1 wiki page/g' \
      -e 's/docs\/([A-Za-z0-9_-]+)\.md/the \1 wiki page/g' "$f"
  done
sed -i 's|<code>docs/GE-ENFORCE.md</code>|the GE-ENFORCE page in the project wiki|' \
  frontend/src/views/geenforce/ManifestEditor.vue 2>/dev/null || true
# internal infra never named on GitHub; this repo's own URL maps to the
# real GitHub home, anything else degrades to a placeholder.
GITHUB_URL='https://github.com/ge-aero/shopdb-flask'
{ grep -rlZ 'gitea\.proudtech\.net' . 2>/dev/null || true; } | while IFS= read -r -d '' f; do
  sed -i -e "s|https://gitea\.proudtech\.net/ge-aerospace/shopdb-flask|$GITHUB_URL|g" \
         -e 's|gitea\.proudtech\.net|<git-host>|g' "$f"
done
{ grep -rlZi 'gitea' --exclude-dir=.git . 2>/dev/null || true; } | while IFS= read -r -d '' f; do
  sed -i -e 's/the GE Aerospace Gitea/the internal GE Aerospace git server/g' \
         -e 's/Gitea Actions CI/CI/g' \
         -e 's/Gitea Actions/CI/g' "$f"
done
# CHANGELOG compare/release link definitions reference tags that do not
# exist on GitHub (history is squashed there) - drop them.
sed -i '/^\[[^]]*\]: .*\/\(compare\|releases\)\//d' CHANGELOG.md
# frontend/CLAUDE.md publishes under a neutral name
if [ -f "$WORK/frontend/CLAUDE.md" ]; then
  cp "$WORK/frontend/CLAUDE.md" frontend/DEVELOPMENT-STANDARDS.md
fi
sed -i "s/rootpassword/changeme/g" shopdb/config.py 2>/dev/null || true

# --- 3. scrub gate: refuse to commit if anything internal leaks ---
# The site patterns are ADR-015's, and they belong here as much as the internal
# tooling names do: this repository is public and the product is multi-site, so
# one plant's server name or internal networks reaching it is both a disclosure
# and a lie to every other site. They were absent, and a bundled plugin shipped
# a production UNC path to GitHub for a fortnight before anyone noticed.
LEAKS=$(grep -rlIiE 'claude|anthropic|fable 5|gitea|proudtech|home/camp|rootpassword|tsgwp00525|wjs\.geaerospace\.net|10\.134\.48\.|10\.48\.249\.' \
  --exclude-dir=.git . || true)
if [ -n "$LEAKS" ]; then
  echo "SCRUB GATE FAILED - internal references in:"; echo "$LEAKS"; exit 1
fi

# --- 4. commit (no-op safe) ---
git add -A
if git diff --cached --quiet; then
  echo "no changes vs publication tree - nothing to export"
else
  git commit -m "$MSG"
fi
git log --oneline -3

# --- 5. bundle for transfer (full bundle: stateless, fetch takes only new) ---
#
# ONE BUNDLE PER BRANCH, named after it.
#
# This wrote a single shopdb-flask-pub.bundle whatever it published, which was
# fine while only main was ever published and became a trap the moment it was
# not: exporting a site branch silently overwrote the bundle holding main's
# work, and the work PC's documented `git fetch <bundle> main` then failed
# because that ref was no longer in the file. Two branches cannot share one
# filename.
BUNDLE="$OUT/shopdb-flask-pub-$BRANCH.bundle"
git bundle create "$BUNDLE" HEAD "$BRANCH" --tags
git bundle verify "$BUNDLE" >/dev/null
echo "bundle: $BUNDLE"

# --- 6. optional per-instance dists (both bases: /ops = dev, /shopdb = prod) ---
# Two instances run on the box; each needs its own base-path build. Always
# rebuild BOTH so prod never ships a stale frontend.
if [ "$BUILD_DIST" = 1 ]; then
  cd "$WORK/frontend"
  for base in ops shopdb; do
    VITE_BASE_PATH=/$base/ npm run build --silent
    rm -rf "$OUT/frontend-dist-subpath-$base"
    cp -r dist "$OUT/frontend-dist-subpath-$base"
    echo "dist:   $OUT/frontend-dist-subpath-$base/  (base /$base/)"
  done
fi

cat <<EOF

On the work PC (in the shopdb-flask-pub clone):

  The FIRST time this branch goes over - it does not exist there yet, so
  there is nothing to check out and nothing to merge into:

    git fetch <path-to>\shopdb-flask-pub-$BRANCH.bundle $BRANCH
    git checkout -b $BRANCH FETCH_HEAD
    git push -u origin $BRANCH

  Every time after that:

    git checkout $BRANCH
    git fetch <path-to>\shopdb-flask-pub-$BRANCH.bundle $BRANCH
    git merge --ff-only FETCH_HEAD
    git push origin $BRANCH
EOF

if [ "$BRANCH" != "main" ]; then
  cat <<EOF

NOTE: this published $BRANCH, not main. Other sites read main, so nothing here
reaches them until these commits are promoted:

  tools/wj-status.sh
  tools/promote-to-main.sh <sha> ...
EOF
fi