shopdb-flask/scripts/check-gates-ran.sh

#!/usr/bin/env bash
#
# Prove the gates actually RAN.
#
# Several checks skip themselves when the file they read is absent:
#
#   tests/test_api_inventory_parity.py   skipif docs/api-inventory.json missing
#   tests/test_openapi_spec.py           skipif docs/openapi.json missing
#   tests/test_docs_publishable.py       skipif docs/*.md missing
#
# That is correct in a published checkout, where docs/ becomes the wiki and
# there is genuinely nothing to check. It was silently wrong on GitHub, which
# runs against exactly that checkout: the modules skipped, CI reported green,
# and an outside branch added nine routes documented nowhere and passed.
#
# So the list is split. Some gates must run everywhere; the docs-dependent ones
# must run HERE, where the files exist, and are allowed to be absent there.
#
# A gate that stops running is worse than no gate: nobody looks at it again.
# This asserts each named module reported at least one PASS and no SKIPS.
#
# Usage:
#   bash scripts/check-gates-ran.sh            # runs the gates itself
#   bash scripts/check-gates-ran.sh report.txt # reads an existing -rs report
set -uo pipefail

cd "$(dirname "$0")/.."

# Gates that must run in EVERY checkout, published or internal. A skip here is
# always a fault.
GATES_EVERYWHERE=(
  "tests/test_openapi_spec.py"
  "tests/test_api_inventory_parity.py"
  "tests/test_core/test_public_read_surface.py"
  "tests/test_core/test_authz.py"
  "tests/test_plugin_migrations.py"
)

# Gates that need files the publication export deliberately leaves behind -
# docs/*.md becomes the wiki, so a published checkout genuinely has nothing to
# check and skipping is correct there. In THIS repo, skipping means something
# broke.
#
# The inventory parity gate used to be here, and that was the hole it exists to
# close. docs/api-inventory.json is not wiki prose: it is a generated artifact,
# tracked exactly like docs/openapi.json beside it, because the app publishes the
# spec built from it. Keying the gate to prose meant a checkout with no docs/*.md
# - which is every fresh clone and every worktree - called itself published and
# ran the route check nowhere. That is where 24 undocumented routes came in.
GATES_INTERNAL_ONLY=(
  "tests/test_docs_publishable.py"
)

# A published checkout is one with no docs/ prose. openapi.json alone does not
# make it internal: that file ships because the app serves it at runtime.
#
# NOTE: docs/wiki-drafts/*.md does NOT count here, and that is a standing
# question rather than an oversight - see the comment on the skipif in
# tests/test_docs_publishable.py. Keep the two definitions identical.
if compgen -G "docs/*.md" > /dev/null; then
  GATES=("${GATES_EVERYWHERE[@]}" "${GATES_INTERNAL_ONLY[@]}")
  echo "==> Internal checkout: every gate must run"
else
  GATES=("${GATES_EVERYWHERE[@]}")
  echo "==> Published checkout: docs-dependent gates are expected to be absent"
fi

PYTHON="${PYTHON:-python}"
[ -x venv/bin/python ] && PYTHON=venv/bin/python

# REFUSE TO RUN WITHOUT A REAL INTERPRETER. In a git worktree there is no venv,
# `python` is often not on PATH at all, and the pytest line below ends in
# `|| true` - so the run died 127, the report was a single "command not found"
# line, every gate matched nothing, and the script still printed "All gates ran."
# and exited 0. A gate-checker that passes when it ran nothing is worse than no
# gate-checker, because its exit code is what everything else trusts.
if [ -z "${1:-}" ] && ! "$PYTHON" -c 'import pytest' >/dev/null 2>&1; then
  echo "FAIL: '$PYTHON' cannot import pytest, so no gate can run here." >&2
  echo "      Set PYTHON=/path/to/python (a venv that has pytest) and re-run." >&2
  exit 1
fi

report="${1:-}"
if [ -z "$report" ]; then
  report="$(mktemp)"
  # -rs lists every skip with its reason, which is the thing being checked.
  # -rf and -rE add the "FAILED <nodeid>" and "ERROR <nodeid>" lines, which -rs
  # alone does NOT emit: this script grepped for them anyway, so on a real
  # failure it printed its header and then nothing at all.
  "$PYTHON" -m pytest "${GATES[@]}" -q -rsfE > "$report" 2>&1 || true
fi

echo "==> Checking that every gate ran"

failed=0
for gate in "${GATES[@]}"; do
  if [ ! -f "$gate" ]; then
    echo "  MISSING  $gate - the file is not in this checkout"
    failed=1
    continue
  fi
  if grep -q "SKIPPED.*$(basename "$gate")" "$report"; then
    reason=$(grep "SKIPPED.*$(basename "$gate")" "$report" | head -1 | sed 's/.*-- //')
    echo "  SKIPPED  $gate"
    echo "           reason: ${reason:-not given}"
    failed=1
    continue
  fi
  echo "  ran      $gate"
done

# Name what failed, from whatever the report actually contains. This script can
# be handed a report somebody else produced, and that one may have been run
# without -rf, so the "FAILED <nodeid>" lines are not guaranteed to be there.
# The FAILURES section underlines are: pytest prints "____ test_name ____" above
# every traceback whether the short summary was asked for or not.
list_failures() {
  if grep -qE '^(FAILED|ERROR) ' "$1"; then
    grep -E '^(FAILED|ERROR) ' "$1"
    return
  fi
  if grep -qE '^_{3,} .+ _{3,}$' "$1"; then
    sed -n 's/^_\{3,\} \(.*[^_]\) _\{3,\}$/\1/p' "$1"
    return
  fi
  # Nothing named. Hand over the count line rather than an empty list, so the
  # reader knows to open the report instead of believing there is nothing here.
  grep -E '[0-9]+ (failed|error)' "$1" | tail -1
  echo "(no test names in this report; run without an argument to get them)"
}

# The old condition was "^[0-9]+ failed" or the bare word "error" anywhere,
# which matches a test name, a warning or a docstring. Anchor on the shapes
# pytest actually prints: a short-summary line, a count in the tally, or an
# interrupted collection.
if grep -qE '^(FAILED|ERROR) |(^|, )[0-9]+ (failed|error)|^!+ Interrupted' "$report"; then
  echo
  echo "Gate tests FAILED (not merely skipped):"
  list_failures "$report" | sed 's/^/  /'
  failed=1
fi

echo
if [ "$failed" -ne 0 ]; then
  echo "FAIL: a gate did not run. A skipped gate reports green and checks nothing."
  echo "      If a docs-dependent gate skipped in THIS repo, the file it reads"
  echo "      was not generated: run scripts/gen_openapi.py and"
  echo "      scripts/gen_project_map.py. If one skipped in the PUBLISHED repo,"
  echo "      the export dropped a file the gate needs - see tools/export-github.sh."
  exit 1
fi
echo "All gates ran."