#!/bin/bash # Export the working repo to the GitHub publication repo and emit a bundle. # # Pipeline: working repo (full history, internal refs) -> pruned/scrubbed # tree -> commit in ~/projects/shopdb-flask-pub (the local mirror of what # enterprise GitHub holds) -> full git bundle in /home/camp/pxe-images/ for # transfer to the work PC, which pushes it to GitHub. # # Usage: # tools/export-github.sh "Commit message for the publication commit" # tools/export-github.sh --dist # also rebuild BOTH frontend dists # # (/ops = dev, /shopdb = prod) # # This script lives in tools/, which is itself excluded from publication. set -euo pipefail WORK=/home/camp/projects/shopdb-flask PUB=/home/camp/projects/shopdb-flask-pub OUT=/home/camp/pxe-images/github BUILD_DIST=0 MSG="" for arg in "$@"; do case "$arg" in --dist) BUILD_DIST=1 ;; *) MSG="$arg" ;; esac done [ -n "$MSG" ] || { echo "usage: $0 [--dist] \"commit message\""; exit 1; } [ -d "$PUB/.git" ] || { echo "publication repo missing at $PUB"; exit 1; } # --- 0. which branch is being published ------------------------------------- # # This syncs the WORKING TREE, not a branch, and it used to commit whatever it # found onto the publication repo's main and print "git push origin main". So # running it while checked out on a site branch published that site's work AS # THE PRODUCT, to the repository every other site consumes, without saying a # word about it. # # The publication repo now mirrors the branch you are actually on. BRANCH="$(git -C "$WORK" rev-parse --abbrev-ref HEAD)" if [ "$BRANCH" = "HEAD" ]; then echo "detached HEAD in $WORK - check out a branch before exporting" >&2 exit 1 fi echo "==> publishing branch: $BRANCH" # --- 1. sync the tree (working -> pub), minus everything never published --- # # NOTE the deploy/windows/installer/bundle exclude below. That is ~100MB of build # output holding a COPY of the whole application tree plus the wheels and vendor # installers. It is gitignored, but this sync walks the WORKING TREE rather than # git, so it came through anyway - and its copies of config.py and # requirements.txt then tripped the scrub gate. Anything else generated into the # working tree needs excluding here too, for the same reason. # rsync --exclude also PROTECTS a path from --delete, so anything that reached # the publication tree before its exclude existed stays there forever, invisible # to the sync and caught only by the scrub gate. Purge the generated paths first # so adding an exclude is enough on its own. # # docs/ does NOT publish - the prose becomes the WIKI instead. One exception, # and it is not a documentation decision: # # docs/openapi.json is read AT RUNTIME by shopdb/core/api/docs.py, which # serves the /api/docs page every published site ships. Excluding it meant # that page loaded and then answered {"error":"openapi.json not generated"} # on every site installed from the published repo. It is a product file that # happens to live under docs/, not documentation. # # It also un-skips tests/test_openapi_spec.py, which had been reporting green # by not running - see scripts/check-gates-ran.sh. # # docs/api-inventory.json stays OUT: nothing reads it at runtime, so it is a # development gate and belongs with the wiki. That does mean # test_api_inventory_parity cannot run on GitHub, which is why the gate guard # separates the checks that must run everywhere from the ones that only run # where the repo is complete. # instance/plugins.json is one site's plugin CHOICES. It is excluded, so it was # never published - but an old copy sat in the publication tree for months, # protected from --delete by that same exclude, and it made the published repo's # own test suite fail locally: 13 plugins enabled where the code expects 16. # A publication tree that cannot pass its own tests is not one anybody can # verify before shipping. for stale in .pytest_cache .ruff_cache htmlcov .coverage instance/plugins.json \ deploy/windows/installer/bundle deploy/windows/installer/Output \ deploy/windows/installer/plugins.iss deploy/windows/installer/version.iss; do rm -rf "${PUB:?}/$stale" done # .env is excluded by name, but a stray copy BESIDE it is not: a hand-made # .env.bak- was picked up by this sync, because it walks the WORKING TREE # and .gitignore does not apply to it. It got no further than the scrub gate, # which is what that gate is for, but a file holding live credentials should # never reach the publication tree at all. Hence --exclude '.env.*', with # --include '.env.example' BEFORE it: rsync takes the first matching rule, and # .env.example does publish. # # Keep flags on their own lines. A comment inside this backslash-continued # command is not a comment - bash hands the text to rsync as arguments, and it # dies with a usage dump rather than anything that names the real problem. # Mirror the branch in the publication repo BEFORE the sync. # # Order matters and got this wrong once: done after the rsync, `git checkout # --force` reverts the tree to the branch's committed state and throws away # everything rsync just wrote, so the export reports "no changes" and publishes # nothing. It is a silent no-op, which is the worst shape for this failure - # the script still prints a bundle path and looks like it worked. # # --force is needed because the tree may hold leftovers from the previously # published branch, and safe because this tree is a mirror: the rsync below # replaces it wholesale from $WORK. ( cd "$PUB" if git show-ref --verify --quiet "refs/heads/$BRANCH"; then git checkout --force "$BRANCH" >/dev/null else # A new site branch starts from main, as it does in the dev repo, so its # history is the product's plus whatever it adds. echo "==> publication repo: creating branch $BRANCH from main" git checkout --force -b "$BRANCH" main >/dev/null fi ) # NOTE ON EXCLUDES: this walks the WORKING TREE, so .gitignore does not apply. # Anything ignored-but-present has to be named here or it reaches publication - # The agent's .NET build output (agent/*/obj, agent/*/bin) is gitignored but # rsync walks the WORKING TREE, so a local `dotnet build` puts absolute host # paths into project.assets.json and friends and the scrub gate stops the whole # export. Excluded here rather than left to whoever built the agent last to # remember to clean up. Same shape as the .env.bak below. # # how a stray .env.bak once did, and how devnotes/ (local review writeups, chat # handoffs) did. The scrub gate catches such things, which is the point, but a # gate failure is a stop, not a filter: add the exclude rather than deleting # your notes to get a bundle out. rsync -a --delete \ --exclude '.git' \ --exclude '.gitea' \ --include '/docs/' \ --include '/docs/openapi.json' \ --exclude '/docs/*' \ --exclude '/tools' \ --exclude '/mcp' \ --exclude '/devnotes' \ --exclude 'start-api.sh' \ --exclude 'start-ui.sh' \ --exclude 'CLAUDE.md' \ --exclude 'frontend/CLAUDE.md' \ --exclude 'tests/test_docs_contract.py' \ --exclude 'tests/test_plugins/test_geenforce_parity.py' \ --exclude 'tests/test_plugins/test_zabbix_live.py' \ --exclude 'venv' \ --exclude 'node_modules' \ --exclude 'agent/*/obj' \ --exclude 'agent/*/bin' \ --exclude 'deploy/windows/installer/bundle' \ --exclude 'deploy/windows/installer/Output' \ --exclude 'deploy/windows/installer/plugins.iss' \ --exclude 'deploy/windows/installer/version.iss' \ --exclude 'frontend/dist*' \ --exclude 'frontend/src/.plugins-staged' \ --exclude 'frontend/src/router/routes.gen.js' \ --exclude '/instance' \ --include '.env.example' \ --exclude '.env.*' \ --exclude '.env' \ --exclude '__pycache__' \ --exclude '.pytest_cache' \ --exclude '.ruff_cache' \ --exclude '.coverage' \ --exclude 'htmlcov' \ --exclude '*.pyc' \ --exclude 'scripts/site_imports/wjf/idmap.json' \ "$WORK/" "$PUB/" # --- 1b. every bundled plugin must have survived the sync --- # # The excludes above are rsync patterns, and a pattern with no leading slash # matches at ANY depth. 'tools' was meant to drop the repo-root tools/ dir (this # script lives in it) and silently dropped plugins/tools/ as well, so a plugin # was committed, exported, bundled and deployed without ever being in the # payload - the failure only surfaced as a missing manifest.json on the server. # The excludes are anchored now; this check is what makes a recurrence loud. missing="" for manifest in "$WORK"/plugins/*/manifest.json; do [ -e "$manifest" ] || continue plugin=$(basename "$(dirname "$manifest")") [ -f "$PUB/plugins/$plugin/manifest.json" ] || missing="$missing $plugin" done if [ -n "$missing" ]; then echo "ERROR: bundled plugin(s) missing from the publication tree:$missing" >&2 echo " an rsync --exclude above is matching them; anchor it with a leading slash." >&2 exit 1 fi cd "$PUB" # --- 2. re-apply the publication wording (idempotent) --- # docs/ lives only in the wiki on the GitHub side. # NOTE the `|| true` on each scrub grep below. grep exits 1 when it matches # NOTHING, and under `set -euo pipefail` that failing pipeline aborts the whole # script - silently, because grep prints nothing to say so. That is exactly what # happened once every internal reference had been scrubbed: the gitea grep found # no matches, the export died before committing or rebuilding the dists, and it # exited without a single line of output. A scrub that finds nothing left to # scrub is the SUCCESS case, so it must not be an error. { grep -rlZ 'docs/' --include='*.py' --include='*.md' --include='*.sh' \ --include='*.tmpl' --include='*.vue' --include='*.example' . 2>/dev/null || true; } | while IFS= read -r -d '' f; do case "$f" in ./CHANGELOG.md) continue ;; esac sed -i -E \ -e 's/\[`?docs\/([A-Za-z0-9_-]+)\.md`?\]\((\.\.\/)*docs\/[A-Za-z0-9_-]+\.md\)/the \1 page in the project wiki/g' \ -e 's/docs\/proposals\/ge-enforce-plugin\.md/the ge-enforce-plugin proposal in the project wiki/g' \ -e 's/`docs\/([A-Za-z0-9_-]+)\.md`/the \1 wiki page/g' \ -e 's/docs\/([A-Za-z0-9_-]+)\.md/the \1 wiki page/g' "$f" done sed -i 's|docs/GE-ENFORCE.md|the GE-ENFORCE page in the project wiki|' \ frontend/src/views/geenforce/ManifestEditor.vue 2>/dev/null || true # internal infra never named on GitHub; this repo's own URL maps to the # real GitHub home, anything else degrades to a placeholder. GITHUB_URL='https://github.com/ge-aero/shopdb-flask' { grep -rlZ 'gitea\.proudtech\.net' . 2>/dev/null || true; } | while IFS= read -r -d '' f; do sed -i -e "s|https://gitea\.proudtech\.net/ge-aerospace/shopdb-flask|$GITHUB_URL|g" \ -e 's|gitea\.proudtech\.net||g' "$f" done { grep -rlZi 'gitea' --exclude-dir=.git . 2>/dev/null || true; } | while IFS= read -r -d '' f; do sed -i -e 's/the GE Aerospace Gitea/the internal GE Aerospace git server/g' \ -e 's/Gitea Actions CI/CI/g' \ -e 's/Gitea Actions/CI/g' "$f" done # CHANGELOG compare/release link definitions reference tags that do not # exist on GitHub (history is squashed there) - drop them. sed -i '/^\[[^]]*\]: .*\/\(compare\|releases\)\//d' CHANGELOG.md # frontend/CLAUDE.md publishes under a neutral name if [ -f "$WORK/frontend/CLAUDE.md" ]; then cp "$WORK/frontend/CLAUDE.md" frontend/DEVELOPMENT-STANDARDS.md fi sed -i "s/rootpassword/changeme/g" shopdb/config.py 2>/dev/null || true # --- 3. scrub gate: refuse to commit if anything internal leaks --- # The site patterns are ADR-015's, and they belong here as much as the internal # tooling names do: this repository is public and the product is multi-site, so # one plant's server name or internal networks reaching it is both a disclosure # and a lie to every other site. They were absent, and a bundled plugin shipped # a production UNC path to GitHub for a fortnight before anyone noticed. LEAKS=$(grep -rlIiE 'claude|anthropic|fable 5|gitea|proudtech|home/camp|rootpassword|tsgwp00525|wjs\.geaerospace\.net|10\.134\.48\.|10\.48\.249\.' \ --exclude-dir=.git . || true) if [ -n "$LEAKS" ]; then echo "SCRUB GATE FAILED - internal references in:"; echo "$LEAKS"; exit 1 fi # --- 4. commit (no-op safe) --- git add -A if git diff --cached --quiet; then echo "no changes vs publication tree - nothing to export" else git commit -m "$MSG" fi git log --oneline -3 # --- 5. bundle for transfer (full bundle: stateless, fetch takes only new) --- # # ONE BUNDLE PER BRANCH, named after it. # # This wrote a single shopdb-flask-pub.bundle whatever it published, which was # fine while only main was ever published and became a trap the moment it was # not: exporting a site branch silently overwrote the bundle holding main's # work, and the work PC's documented `git fetch main` then failed # because that ref was no longer in the file. Two branches cannot share one # filename. BUNDLE="$OUT/shopdb-flask-pub-$BRANCH.bundle" git bundle create "$BUNDLE" HEAD "$BRANCH" --tags git bundle verify "$BUNDLE" >/dev/null echo "bundle: $BUNDLE" # --- 6. the frontend dist for THIS branch's instance --------------------------- # # ONE BRANCH, ONE INSTANCE, ONE DIST. The rule for this site: # # wj -> /shopdb (WJ PRODUCTION) dist: frontend-dist-subpath-shopdb # dev -> /ops (dev instance) dist: frontend-dist-subpath-ops # main -> nothing (curated release other sites adopt) # # This used to build BOTH dists and refuse --dist unless you were on main, # because production was assumed to install main. It does not: production # installs wj. Under the old rule a wj export failed at this step and the # dists on the share came from whichever main export happened to run next - # so the code going to prod and the frontend going to prod came from different # branches, silently, and the deploy scripts could not tell. # # Each dist now carries a .dist-source marker naming the branch, the commit and # the time that built it. update-prod-server.cmd and update-dev-server.cmd read # it and refuse a dist built from the wrong branch, so the trap the old comment # worried about is closed by a check rather than by a convention. case "$BRANCH" in wj) DIST_BASE=shopdb ;; dev) DIST_BASE=ops ;; *) DIST_BASE='' ;; esac if [ "$BUILD_DIST" = 1 ] && [ -z "$DIST_BASE" ]; then echo "REFUSED: --dist on branch '$BRANCH'." >&2 echo >&2 echo "Only a branch that a server actually runs builds a dist:" >&2 echo " wj -> /shopdb (production)" >&2 echo " dev -> /ops" >&2 echo >&2 echo "'$BRANCH' deploys nowhere, so a dist built from it could only ever be" >&2 echo "installed by mistake. Export it without --dist." >&2 exit 1 fi if [ "$BUILD_DIST" = 1 ]; then cd "$WORK/frontend" VITE_BASE_PATH=/$DIST_BASE/ npm run build --silent rm -rf "$OUT/frontend-dist-subpath-$DIST_BASE" cp -r dist "$OUT/frontend-dist-subpath-$DIST_BASE" # Provenance, read by the deploy scripts. A dist is a directory of hashed # filenames with nothing in it that says where it came from, and robocopy # /MIR will happily mirror the wrong one onto a correctly-identified server. { echo "branch=$BRANCH" echo "commit=$(git -C "$WORK" rev-parse HEAD)" echo "base=/$DIST_BASE/" echo "built=$(date '+%Y-%m-%d %H:%M:%S %z')" echo "host=$(hostname)" } > "$OUT/frontend-dist-subpath-$DIST_BASE/.dist-source" echo "dist: $OUT/frontend-dist-subpath-$DIST_BASE/ (base /$DIST_BASE/, from $BRANCH)" fi cat <\shopdb-flask-pub-$BRANCH.bundle $BRANCH git checkout -b $BRANCH FETCH_HEAD git push -u origin $BRANCH Every time after that: git checkout $BRANCH git fetch \shopdb-flask-pub-$BRANCH.bundle $BRANCH git merge --ff-only FETCH_HEAD git push origin $BRANCH EOF if [ "$BRANCH" != "main" ]; then cat < ... EOF fi