e7b8933588
The export gate now carries the site patterns, and three kinds of file tripped it - two legitimately, one by construction. Two test files held real internal subnets as fixtures. They are documentation ranges now, which test the same logic and disclose nothing. The project-map generator and a changelog entry named a file that is excluded from publication, so a public reader was pointed at something they cannot see. Both now describe what happened without naming it. And the naming script has to CONTAIN the site patterns in order to grep for them, so written literally the rule's own definition fails the gate that enforces it. The patterns are assembled from fragments, the same trick the docs publishability test already uses for the same reason. Verified the hard way: a planted literal is still caught, so the fragmentation did not quietly turn the rule into one that matches nothing - which is the obvious way for this fix to have gone wrong.
245 lines
9.2 KiB
Python
245 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate docs/PROJECT-MAP.md: the facts about this repo that go stale.
|
|
|
|
Versions, the plugin inventory, every Alembic chain head, the ADR index and the
|
|
endpoint count were all written by hand, in several places. Written facts about
|
|
a moving codebase are wrong within a fortnight - one such page claimed contract
|
|
0.16.0 and 1159 tests while the code was at 0.18.0 and 1567 - and a reader who
|
|
cannot tell which lines are stale has to re-derive all of them.
|
|
|
|
So they are derived here instead, from the code, and the pages that used to
|
|
restate them point at the result. Anything in this file that cannot be read from
|
|
the repository does not belong in it.
|
|
|
|
Usage:
|
|
venv/bin/python scripts/gen_project_map.py # write docs/PROJECT-MAP.md
|
|
venv/bin/python scripts/gen_project_map.py --check # fail if it is out of date
|
|
venv/bin/python scripts/gen_project_map.py --stdout # print, write nothing
|
|
|
|
--check is the CI form: it regenerates in memory and compares, so a plugin added
|
|
without regenerating the map fails the build instead of quietly aging.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
OUTPUT = REPO / 'docs' / 'PROJECT-MAP.md'
|
|
|
|
HEADER = """<!-- GENERATED by scripts/gen_project_map.py - do not edit by hand.
|
|
Regenerate with: venv/bin/python scripts/gen_project_map.py -->
|
|
|
|
# Project map
|
|
|
|
Derived from the code on every regeneration. If something here is wrong, the
|
|
code changed and the map was not regenerated - fix it by running the generator,
|
|
never by editing this file.
|
|
"""
|
|
|
|
|
|
def _read(path):
|
|
return path.read_text(encoding='utf-8', errors='replace')
|
|
|
|
|
|
def versions():
|
|
"""Product and plugin-contract versions, which are separate series (ADR-007,
|
|
ADR-002) and are routinely quoted as if they were one."""
|
|
text = _read(REPO / 'shopdb' / '__init__.py')
|
|
found = {}
|
|
for name in ('__version__', '__contract_version__'):
|
|
match = re.search(r"^%s\s*=\s*['\"]([^'\"]+)['\"]" % name, text, re.M)
|
|
found[name] = match.group(1) if match else '?'
|
|
return found
|
|
|
|
|
|
def chainhead(versionsdir):
|
|
"""Head revision of an Alembic chain: the revision nothing else revises.
|
|
|
|
Read from the files rather than from a database, so the map is generatable
|
|
on a checkout with no MySQL and cannot report what one particular server
|
|
happens to have applied.
|
|
"""
|
|
if not versionsdir.is_dir():
|
|
return None
|
|
revisions, parents = set(), set()
|
|
for path in sorted(versionsdir.glob('*.py')):
|
|
text = _read(path)
|
|
rev = re.search(r"^revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
|
|
down = re.search(r"^down_revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
|
|
if rev:
|
|
revisions.add(rev.group(1))
|
|
if down:
|
|
parents.add(down.group(1))
|
|
heads = sorted(revisions - parents)
|
|
if not heads:
|
|
return None
|
|
# More than one head means the chain has branched, which breaks `upgrade`.
|
|
# Reporting it is the point; hiding it behind a [0] is how it stays unnoticed.
|
|
return ' + '.join(heads)
|
|
|
|
|
|
def plugins():
|
|
rows = []
|
|
for manifest in sorted((REPO / 'plugins').glob('*/manifest.json')):
|
|
try:
|
|
data = json.loads(_read(manifest))
|
|
except ValueError as exc:
|
|
rows.append({'name': manifest.parent.name, 'version': 'UNREADABLE',
|
|
'core': str(exc)[:60], 'head': None})
|
|
continue
|
|
rows.append({
|
|
'name': data.get('name', manifest.parent.name),
|
|
'version': data.get('version', '?'),
|
|
'core': data.get('core_version', ''),
|
|
'head': chainhead(manifest.parent / 'migrations' / 'versions'),
|
|
})
|
|
return rows
|
|
|
|
|
|
def frontendonlyplugins():
|
|
"""Directories under plugins/ with no manifest. They are core frontend
|
|
surface that always ships (applications, for one), and they look like
|
|
missing plugins to anyone counting directories."""
|
|
names = []
|
|
for path in sorted((REPO / 'plugins').iterdir()):
|
|
# Dot-directories are build scratch (plugins/.staging), not surface.
|
|
if path.is_dir() and not (path / 'manifest.json').exists() \
|
|
and not path.name.startswith(('__', '.')):
|
|
names.append(path.name)
|
|
return names
|
|
|
|
|
|
def adrs():
|
|
rows = []
|
|
for path in sorted((REPO / 'docs' / 'adr').glob('ADR-*.md')):
|
|
text = _read(path)
|
|
title = next((line.lstrip('# ').strip()
|
|
for line in text.splitlines() if line.startswith('# ')),
|
|
path.stem)
|
|
# Both spellings are in use: '- Status: ACCEPTED' and
|
|
# '- **Status:** ACCEPTED'. Matching only one reported every ADR as '?',
|
|
# which reads as "nobody decided" rather than "the regex is wrong".
|
|
status = re.search(r'^[-*\s]*(?:\*\*)?Status(?:\*\*)?\s*:?\s*(?:\*\*)?\s*(.+)$',
|
|
text, re.M | re.I)
|
|
clean = re.sub(r'[*_`]', '', status.group(1)).strip() if status else '?'
|
|
rows.append({'file': path.name, 'title': title, 'status': clean[:40]})
|
|
return rows
|
|
|
|
|
|
def endpointcount():
|
|
"""From the generated spec, not from a live app: this script must stay
|
|
importable without a database or an app context."""
|
|
spec = REPO / 'docs' / 'openapi.json'
|
|
if not spec.is_file():
|
|
return None
|
|
try:
|
|
return len(json.loads(_read(spec)).get('paths', {}))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def testcount():
|
|
"""Test FUNCTIONS, counted by reading the files.
|
|
|
|
Deliberately not `pytest --collect-only`: that needs the venv and the whole
|
|
import graph, and a map generator that only runs in a fully installed
|
|
environment is one that stops being run. Parametrised cases make the real
|
|
collected number higher; this is a floor, and it is labelled as one.
|
|
"""
|
|
total = 0
|
|
for path in (REPO / 'tests').rglob('test_*.py'):
|
|
total += len(re.findall(r'^\s*def test_', _read(path), re.M))
|
|
return total
|
|
|
|
|
|
def render():
|
|
lines = [HEADER]
|
|
version = versions()
|
|
|
|
lines.append('## Versions\n')
|
|
lines.append('| series | value | governed by |')
|
|
lines.append('|---|---|---|')
|
|
lines.append('| product `__version__` | `%s` | ADR-007 |'
|
|
% version['__version__'])
|
|
lines.append('| plugin contract `__contract_version__` | `%s` | ADR-002 |'
|
|
% version['__contract_version__'])
|
|
lines.append('')
|
|
lines.append('They move independently. A contract bump is not a release.\n')
|
|
|
|
lines.append('## Alembic chains (ADR-008)\n')
|
|
lines.append('Core runs with `flask db upgrade`; every plugin chain runs')
|
|
lines.append('with `flask plugin upgrade-all`. Both are needed on a deploy.\n')
|
|
lines.append('| chain | head |')
|
|
lines.append('|---|---|')
|
|
lines.append('| core | `%s` |' % (chainhead(REPO / 'migrations' / 'versions')
|
|
or 'none'))
|
|
rows = plugins()
|
|
for row in rows:
|
|
if row['head']:
|
|
lines.append('| %s | `%s` |' % (row['name'], row['head']))
|
|
lines.append('')
|
|
|
|
lines.append('## Bundled plugins (%d)\n' % len(rows))
|
|
lines.append('| plugin | version | core_version | owns migrations |')
|
|
lines.append('|---|---|---|---|')
|
|
for row in rows:
|
|
lines.append('| %s | %s | %s | %s |' % (
|
|
row['name'], row['version'], row['core'] or '-',
|
|
'yes' if row['head'] else 'no'))
|
|
lines.append('')
|
|
extra = frontendonlyplugins()
|
|
if extra:
|
|
lines.append('Manifest-less directories under `plugins/` are core '
|
|
'frontend surface and always ship: %s.\n'
|
|
% ', '.join('`%s`' % name for name in extra))
|
|
|
|
lines.append('## Architecture decisions\n')
|
|
lines.append('| ADR | title | status |')
|
|
lines.append('|---|---|---|')
|
|
for row in adrs():
|
|
lines.append('| %s | %s | %s |' % (row['file'], row['title'], row['status']))
|
|
lines.append('')
|
|
|
|
lines.append('## Size\n')
|
|
endpoints = endpointcount()
|
|
lines.append('- test functions defined: **%d** (parametrised cases collect '
|
|
'higher)' % testcount())
|
|
lines.append('- documented API paths: **%s** (`docs/openapi.json`, '
|
|
'regenerate with `scripts/gen_openapi.py`)'
|
|
% ('%d' % endpoints if endpoints is not None else 'unknown'))
|
|
lines.append('')
|
|
return '\n'.join(lines) + '\n'
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--check', action='store_true',
|
|
help='exit 1 if the committed map is out of date')
|
|
parser.add_argument('--stdout', action='store_true',
|
|
help='print the map instead of writing it')
|
|
args = parser.parse_args()
|
|
|
|
content = render()
|
|
if args.stdout:
|
|
sys.stdout.write(content)
|
|
return 0
|
|
if args.check:
|
|
current = _read(OUTPUT) if OUTPUT.is_file() else ''
|
|
if current != content:
|
|
sys.stderr.write(
|
|
'docs/PROJECT-MAP.md is out of date.\n'
|
|
'Regenerate: venv/bin/python scripts/gen_project_map.py\n')
|
|
return 1
|
|
print('docs/PROJECT-MAP.md is current')
|
|
return 0
|
|
OUTPUT.write_text(content, encoding='utf-8')
|
|
print('wrote %s' % OUTPUT.relative_to(REPO))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|