Security hardening, multi-site portability, and a documentation and API-surface sweep
This commit is contained in:
244
scripts/gen_project_map.py
Normal file
244
scripts/gen_project_map.py
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the PROJECT-MAP wiki page: the facts about this repo that go stale.
|
||||
|
||||
Versions, the plugin inventory, every Alembic chain head, the ADR index and the
|
||||
endpoint count were all written by hand, in several places. Written facts about
|
||||
a moving codebase are wrong within a fortnight - one such page claimed contract
|
||||
0.16.0 and 1159 tests while the code was at 0.18.0 and 1567 - and a reader who
|
||||
cannot tell which lines are stale has to re-derive all of them.
|
||||
|
||||
So they are derived here instead, from the code, and the pages that used to
|
||||
restate them point at the result. Anything in this file that cannot be read from
|
||||
the repository does not belong in it.
|
||||
|
||||
Usage:
|
||||
venv/bin/python scripts/gen_project_map.py # write the PROJECT-MAP wiki page
|
||||
venv/bin/python scripts/gen_project_map.py --check # fail if it is out of date
|
||||
venv/bin/python scripts/gen_project_map.py --stdout # print, write nothing
|
||||
|
||||
--check is the CI form: it regenerates in memory and compares, so a plugin added
|
||||
without regenerating the map fails the build instead of quietly aging.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = REPO / 'docs' / 'PROJECT-MAP.md'
|
||||
|
||||
HEADER = """<!-- GENERATED by scripts/gen_project_map.py - do not edit by hand.
|
||||
Regenerate with: venv/bin/python scripts/gen_project_map.py -->
|
||||
|
||||
# Project map
|
||||
|
||||
Derived from the code on every regeneration. If something here is wrong, the
|
||||
code changed and the map was not regenerated - fix it by running the generator,
|
||||
never by editing this file.
|
||||
"""
|
||||
|
||||
|
||||
def _read(path):
|
||||
return path.read_text(encoding='utf-8', errors='replace')
|
||||
|
||||
|
||||
def versions():
|
||||
"""Product and plugin-contract versions, which are separate series (ADR-007,
|
||||
ADR-002) and are routinely quoted as if they were one."""
|
||||
text = _read(REPO / 'shopdb' / '__init__.py')
|
||||
found = {}
|
||||
for name in ('__version__', '__contract_version__'):
|
||||
match = re.search(r"^%s\s*=\s*['\"]([^'\"]+)['\"]" % name, text, re.M)
|
||||
found[name] = match.group(1) if match else '?'
|
||||
return found
|
||||
|
||||
|
||||
def chainhead(versionsdir):
|
||||
"""Head revision of an Alembic chain: the revision nothing else revises.
|
||||
|
||||
Read from the files rather than from a database, so the map is generatable
|
||||
on a checkout with no MySQL and cannot report what one particular server
|
||||
happens to have applied.
|
||||
"""
|
||||
if not versionsdir.is_dir():
|
||||
return None
|
||||
revisions, parents = set(), set()
|
||||
for path in sorted(versionsdir.glob('*.py')):
|
||||
text = _read(path)
|
||||
rev = re.search(r"^revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
|
||||
down = re.search(r"^down_revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
|
||||
if rev:
|
||||
revisions.add(rev.group(1))
|
||||
if down:
|
||||
parents.add(down.group(1))
|
||||
heads = sorted(revisions - parents)
|
||||
if not heads:
|
||||
return None
|
||||
# More than one head means the chain has branched, which breaks `upgrade`.
|
||||
# Reporting it is the point; hiding it behind a [0] is how it stays unnoticed.
|
||||
return ' + '.join(heads)
|
||||
|
||||
|
||||
def plugins():
|
||||
rows = []
|
||||
for manifest in sorted((REPO / 'plugins').glob('*/manifest.json')):
|
||||
try:
|
||||
data = json.loads(_read(manifest))
|
||||
except ValueError as exc:
|
||||
rows.append({'name': manifest.parent.name, 'version': 'UNREADABLE',
|
||||
'core': str(exc)[:60], 'head': None})
|
||||
continue
|
||||
rows.append({
|
||||
'name': data.get('name', manifest.parent.name),
|
||||
'version': data.get('version', '?'),
|
||||
'core': data.get('core_version', ''),
|
||||
'head': chainhead(manifest.parent / 'migrations' / 'versions'),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def frontendonlyplugins():
|
||||
"""Directories under plugins/ with no manifest. They are core frontend
|
||||
surface that always ships (applications, for one), and they look like
|
||||
missing plugins to anyone counting directories."""
|
||||
names = []
|
||||
for path in sorted((REPO / 'plugins').iterdir()):
|
||||
# Dot-directories are build scratch (plugins/.staging), not surface.
|
||||
if path.is_dir() and not (path / 'manifest.json').exists() \
|
||||
and not path.name.startswith(('__', '.')):
|
||||
names.append(path.name)
|
||||
return names
|
||||
|
||||
|
||||
def adrs():
|
||||
rows = []
|
||||
for path in sorted((REPO / 'docs' / 'adr').glob('ADR-*.md')):
|
||||
text = _read(path)
|
||||
title = next((line.lstrip('# ').strip()
|
||||
for line in text.splitlines() if line.startswith('# ')),
|
||||
path.stem)
|
||||
# Both spellings are in use: '- Status: ACCEPTED' and
|
||||
# '- **Status:** ACCEPTED'. Matching only one reported every ADR as '?',
|
||||
# which reads as "nobody decided" rather than "the regex is wrong".
|
||||
status = re.search(r'^[-*\s]*(?:\*\*)?Status(?:\*\*)?\s*:?\s*(?:\*\*)?\s*(.+)$',
|
||||
text, re.M | re.I)
|
||||
clean = re.sub(r'[*_`]', '', status.group(1)).strip() if status else '?'
|
||||
rows.append({'file': path.name, 'title': title, 'status': clean[:40]})
|
||||
return rows
|
||||
|
||||
|
||||
def endpointcount():
|
||||
"""From the generated spec, not from a live app: this script must stay
|
||||
importable without a database or an app context."""
|
||||
spec = REPO / 'docs' / 'openapi.json'
|
||||
if not spec.is_file():
|
||||
return None
|
||||
try:
|
||||
return len(json.loads(_read(spec)).get('paths', {}))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def testcount():
|
||||
"""Test FUNCTIONS, counted by reading the files.
|
||||
|
||||
Deliberately not `pytest --collect-only`: that needs the venv and the whole
|
||||
import graph, and a map generator that only runs in a fully installed
|
||||
environment is one that stops being run. Parametrised cases make the real
|
||||
collected number higher; this is a floor, and it is labelled as one.
|
||||
"""
|
||||
total = 0
|
||||
for path in (REPO / 'tests').rglob('test_*.py'):
|
||||
total += len(re.findall(r'^\s*def test_', _read(path), re.M))
|
||||
return total
|
||||
|
||||
|
||||
def render():
|
||||
lines = [HEADER]
|
||||
version = versions()
|
||||
|
||||
lines.append('## Versions\n')
|
||||
lines.append('| series | value | governed by |')
|
||||
lines.append('|---|---|---|')
|
||||
lines.append('| product `__version__` | `%s` | ADR-007 |'
|
||||
% version['__version__'])
|
||||
lines.append('| plugin contract `__contract_version__` | `%s` | ADR-002 |'
|
||||
% version['__contract_version__'])
|
||||
lines.append('')
|
||||
lines.append('They move independently. A contract bump is not a release.\n')
|
||||
|
||||
lines.append('## Alembic chains (ADR-008)\n')
|
||||
lines.append('Core runs with `flask db upgrade`; every plugin chain runs')
|
||||
lines.append('with `flask plugin upgrade-all`. Both are needed on a deploy.\n')
|
||||
lines.append('| chain | head |')
|
||||
lines.append('|---|---|')
|
||||
lines.append('| core | `%s` |' % (chainhead(REPO / 'migrations' / 'versions')
|
||||
or 'none'))
|
||||
rows = plugins()
|
||||
for row in rows:
|
||||
if row['head']:
|
||||
lines.append('| %s | `%s` |' % (row['name'], row['head']))
|
||||
lines.append('')
|
||||
|
||||
lines.append('## Bundled plugins (%d)\n' % len(rows))
|
||||
lines.append('| plugin | version | core_version | owns migrations |')
|
||||
lines.append('|---|---|---|---|')
|
||||
for row in rows:
|
||||
lines.append('| %s | %s | %s | %s |' % (
|
||||
row['name'], row['version'], row['core'] or '-',
|
||||
'yes' if row['head'] else 'no'))
|
||||
lines.append('')
|
||||
extra = frontendonlyplugins()
|
||||
if extra:
|
||||
lines.append('Manifest-less directories under `plugins/` are core '
|
||||
'frontend surface and always ship: %s.\n'
|
||||
% ', '.join('`%s`' % name for name in extra))
|
||||
|
||||
lines.append('## Architecture decisions\n')
|
||||
lines.append('| ADR | title | status |')
|
||||
lines.append('|---|---|---|')
|
||||
for row in adrs():
|
||||
lines.append('| %s | %s | %s |' % (row['file'], row['title'], row['status']))
|
||||
lines.append('')
|
||||
|
||||
lines.append('## Size\n')
|
||||
endpoints = endpointcount()
|
||||
lines.append('- test functions defined: **%d** (parametrised cases collect '
|
||||
'higher)' % testcount())
|
||||
lines.append('- documented API paths: **%s** (`docs/openapi.json`, '
|
||||
'regenerate with `scripts/gen_openapi.py`)'
|
||||
% ('%d' % endpoints if endpoints is not None else 'unknown'))
|
||||
lines.append('')
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--check', action='store_true',
|
||||
help='exit 1 if the committed map is out of date')
|
||||
parser.add_argument('--stdout', action='store_true',
|
||||
help='print the map instead of writing it')
|
||||
args = parser.parse_args()
|
||||
|
||||
content = render()
|
||||
if args.stdout:
|
||||
sys.stdout.write(content)
|
||||
return 0
|
||||
if args.check:
|
||||
current = _read(OUTPUT) if OUTPUT.is_file() else ''
|
||||
if current != content:
|
||||
sys.stderr.write(
|
||||
'the PROJECT-MAP wiki page is out of date.\n'
|
||||
'Regenerate: venv/bin/python scripts/gen_project_map.py\n')
|
||||
return 1
|
||||
print('the PROJECT-MAP wiki page is current')
|
||||
return 0
|
||||
OUTPUT.write_text(content, encoding='utf-8')
|
||||
print('wrote %s' % OUTPUT.relative_to(REPO))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user