Generate the facts that go stale, and record the fortnight in the changelog
CLAUDE.md is read at the start of every session and claimed contract 0.16.0 and 1159 tests while the code was at 0.18.0 and 1567, along with a plugin count and a chain head that were both wrong. Hand-written facts about a moving codebase are wrong within a fortnight, and a reader who cannot tell which lines are stale has to re-derive all of them. So they are derived. scripts/gen_project_map.py reads the versions, the plugin inventory, every Alembic chain head, the ADR index with status and the size of the codebase out of the code, and --check fails a build where the committed map no longer matches. CLAUDE.md now points at it, leads with the multi-site rule, and lists the gates to run instead of restating the conventions they enforce. The changelog's Unreleased section covered 8 of the 46 commits since 0.9.0 and had no Changed, Fixed or Security sections at all. It now carries the whole fortnight, including both contract bumps - which had never been recorded even though ADR-002 makes contract versions their own series.
This commit is contained in:
cproudlock
244
scripts/gen_project_map.py
Normal file
244
scripts/gen_project_map.py
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate docs/PROJECT-MAP.md: the facts about this repo that go stale.
|
||||
|
||||
Versions, the plugin inventory, every Alembic chain head, the ADR index and the
|
||||
endpoint count were all written by hand in CLAUDE.md and in the docs. Written
|
||||
facts about a moving codebase are wrong within a fortnight - CLAUDE.md claimed
|
||||
contract 0.16.0 and 1159 tests while the code was at 0.18.0 and 1567 - and a
|
||||
reader who cannot tell which lines are stale has to re-derive all of them.
|
||||
|
||||
So they are derived here instead, from the code, and CLAUDE.md points at the
|
||||
result. Anything in this file that cannot be read from the repository does not
|
||||
belong in it.
|
||||
|
||||
Usage:
|
||||
venv/bin/python scripts/gen_project_map.py # write docs/PROJECT-MAP.md
|
||||
venv/bin/python scripts/gen_project_map.py --check # fail if it is out of date
|
||||
venv/bin/python scripts/gen_project_map.py --stdout # print, write nothing
|
||||
|
||||
--check is the CI form: it regenerates in memory and compares, so a plugin added
|
||||
without regenerating the map fails the build instead of quietly aging.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = REPO / 'docs' / 'PROJECT-MAP.md'
|
||||
|
||||
HEADER = """<!-- GENERATED by scripts/gen_project_map.py - do not edit by hand.
|
||||
Regenerate with: venv/bin/python scripts/gen_project_map.py -->
|
||||
|
||||
# Project map
|
||||
|
||||
Derived from the code on every regeneration. If something here is wrong, the
|
||||
code changed and the map was not regenerated - fix it by running the generator,
|
||||
never by editing this file.
|
||||
"""
|
||||
|
||||
|
||||
def _read(path):
|
||||
return path.read_text(encoding='utf-8', errors='replace')
|
||||
|
||||
|
||||
def versions():
|
||||
"""Product and plugin-contract versions, which are separate series (ADR-007,
|
||||
ADR-002) and are routinely quoted as if they were one."""
|
||||
text = _read(REPO / 'shopdb' / '__init__.py')
|
||||
found = {}
|
||||
for name in ('__version__', '__contract_version__'):
|
||||
match = re.search(r"^%s\s*=\s*['\"]([^'\"]+)['\"]" % name, text, re.M)
|
||||
found[name] = match.group(1) if match else '?'
|
||||
return found
|
||||
|
||||
|
||||
def chainhead(versionsdir):
|
||||
"""Head revision of an Alembic chain: the revision nothing else revises.
|
||||
|
||||
Read from the files rather than from a database, so the map is generatable
|
||||
on a checkout with no MySQL and cannot report what one particular server
|
||||
happens to have applied.
|
||||
"""
|
||||
if not versionsdir.is_dir():
|
||||
return None
|
||||
revisions, parents = set(), set()
|
||||
for path in sorted(versionsdir.glob('*.py')):
|
||||
text = _read(path)
|
||||
rev = re.search(r"^revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
|
||||
down = re.search(r"^down_revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
|
||||
if rev:
|
||||
revisions.add(rev.group(1))
|
||||
if down:
|
||||
parents.add(down.group(1))
|
||||
heads = sorted(revisions - parents)
|
||||
if not heads:
|
||||
return None
|
||||
# More than one head means the chain has branched, which breaks `upgrade`.
|
||||
# Reporting it is the point; hiding it behind a [0] is how it stays unnoticed.
|
||||
return ' + '.join(heads)
|
||||
|
||||
|
||||
def plugins():
|
||||
rows = []
|
||||
for manifest in sorted((REPO / 'plugins').glob('*/manifest.json')):
|
||||
try:
|
||||
data = json.loads(_read(manifest))
|
||||
except ValueError as exc:
|
||||
rows.append({'name': manifest.parent.name, 'version': 'UNREADABLE',
|
||||
'core': str(exc)[:60], 'head': None})
|
||||
continue
|
||||
rows.append({
|
||||
'name': data.get('name', manifest.parent.name),
|
||||
'version': data.get('version', '?'),
|
||||
'core': data.get('core_version', ''),
|
||||
'head': chainhead(manifest.parent / 'migrations' / 'versions'),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def frontendonlyplugins():
|
||||
"""Directories under plugins/ with no manifest. They are core frontend
|
||||
surface that always ships (applications, for one), and they look like
|
||||
missing plugins to anyone counting directories."""
|
||||
names = []
|
||||
for path in sorted((REPO / 'plugins').iterdir()):
|
||||
# Dot-directories are build scratch (plugins/.staging), not surface.
|
||||
if path.is_dir() and not (path / 'manifest.json').exists() \
|
||||
and not path.name.startswith(('__', '.')):
|
||||
names.append(path.name)
|
||||
return names
|
||||
|
||||
|
||||
def adrs():
|
||||
rows = []
|
||||
for path in sorted((REPO / 'docs' / 'adr').glob('ADR-*.md')):
|
||||
text = _read(path)
|
||||
title = next((line.lstrip('# ').strip()
|
||||
for line in text.splitlines() if line.startswith('# ')),
|
||||
path.stem)
|
||||
# Both spellings are in use: '- Status: ACCEPTED' and
|
||||
# '- **Status:** ACCEPTED'. Matching only one reported every ADR as '?',
|
||||
# which reads as "nobody decided" rather than "the regex is wrong".
|
||||
status = re.search(r'^[-*\s]*(?:\*\*)?Status(?:\*\*)?\s*:?\s*(?:\*\*)?\s*(.+)$',
|
||||
text, re.M | re.I)
|
||||
clean = re.sub(r'[*_`]', '', status.group(1)).strip() if status else '?'
|
||||
rows.append({'file': path.name, 'title': title, 'status': clean[:40]})
|
||||
return rows
|
||||
|
||||
|
||||
def endpointcount():
|
||||
"""From the generated spec, not from a live app: this script must stay
|
||||
importable without a database or an app context."""
|
||||
spec = REPO / 'docs' / 'openapi.json'
|
||||
if not spec.is_file():
|
||||
return None
|
||||
try:
|
||||
return len(json.loads(_read(spec)).get('paths', {}))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def testcount():
|
||||
"""Test FUNCTIONS, counted by reading the files.
|
||||
|
||||
Deliberately not `pytest --collect-only`: that needs the venv and the whole
|
||||
import graph, and a map generator that only runs in a fully installed
|
||||
environment is one that stops being run. Parametrised cases make the real
|
||||
collected number higher; this is a floor, and it is labelled as one.
|
||||
"""
|
||||
total = 0
|
||||
for path in (REPO / 'tests').rglob('test_*.py'):
|
||||
total += len(re.findall(r'^\s*def test_', _read(path), re.M))
|
||||
return total
|
||||
|
||||
|
||||
def render():
|
||||
lines = [HEADER]
|
||||
version = versions()
|
||||
|
||||
lines.append('## Versions\n')
|
||||
lines.append('| series | value | governed by |')
|
||||
lines.append('|---|---|---|')
|
||||
lines.append('| product `__version__` | `%s` | ADR-007 |'
|
||||
% version['__version__'])
|
||||
lines.append('| plugin contract `__contract_version__` | `%s` | ADR-002 |'
|
||||
% version['__contract_version__'])
|
||||
lines.append('')
|
||||
lines.append('They move independently. A contract bump is not a release.\n')
|
||||
|
||||
lines.append('## Alembic chains (ADR-008)\n')
|
||||
lines.append('Core runs with `flask db upgrade`; every plugin chain runs')
|
||||
lines.append('with `flask plugin upgrade-all`. Both are needed on a deploy.\n')
|
||||
lines.append('| chain | head |')
|
||||
lines.append('|---|---|')
|
||||
lines.append('| core | `%s` |' % (chainhead(REPO / 'migrations' / 'versions')
|
||||
or 'none'))
|
||||
rows = plugins()
|
||||
for row in rows:
|
||||
if row['head']:
|
||||
lines.append('| %s | `%s` |' % (row['name'], row['head']))
|
||||
lines.append('')
|
||||
|
||||
lines.append('## Bundled plugins (%d)\n' % len(rows))
|
||||
lines.append('| plugin | version | core_version | owns migrations |')
|
||||
lines.append('|---|---|---|---|')
|
||||
for row in rows:
|
||||
lines.append('| %s | %s | %s | %s |' % (
|
||||
row['name'], row['version'], row['core'] or '-',
|
||||
'yes' if row['head'] else 'no'))
|
||||
lines.append('')
|
||||
extra = frontendonlyplugins()
|
||||
if extra:
|
||||
lines.append('Manifest-less directories under `plugins/` are core '
|
||||
'frontend surface and always ship: %s.\n'
|
||||
% ', '.join('`%s`' % name for name in extra))
|
||||
|
||||
lines.append('## Architecture decisions\n')
|
||||
lines.append('| ADR | title | status |')
|
||||
lines.append('|---|---|---|')
|
||||
for row in adrs():
|
||||
lines.append('| %s | %s | %s |' % (row['file'], row['title'], row['status']))
|
||||
lines.append('')
|
||||
|
||||
lines.append('## Size\n')
|
||||
endpoints = endpointcount()
|
||||
lines.append('- test functions defined: **%d** (parametrised cases collect '
|
||||
'higher)' % testcount())
|
||||
lines.append('- documented API paths: **%s** (`docs/openapi.json`, '
|
||||
'regenerate with `scripts/gen_openapi.py`)'
|
||||
% ('%d' % endpoints if endpoints is not None else 'unknown'))
|
||||
lines.append('')
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--check', action='store_true',
|
||||
help='exit 1 if the committed map is out of date')
|
||||
parser.add_argument('--stdout', action='store_true',
|
||||
help='print the map instead of writing it')
|
||||
args = parser.parse_args()
|
||||
|
||||
content = render()
|
||||
if args.stdout:
|
||||
sys.stdout.write(content)
|
||||
return 0
|
||||
if args.check:
|
||||
current = _read(OUTPUT) if OUTPUT.is_file() else ''
|
||||
if current != content:
|
||||
sys.stderr.write(
|
||||
'docs/PROJECT-MAP.md is out of date.\n'
|
||||
'Regenerate: venv/bin/python scripts/gen_project_map.py\n')
|
||||
return 1
|
||||
print('docs/PROJECT-MAP.md is current')
|
||||
return 0
|
||||
OUTPUT.write_text(content, encoding='utf-8')
|
||||
print('wrote %s' % OUTPUT.relative_to(REPO))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user