shopdb-flask/scripts/gen_project_map.py

#!/usr/bin/env python3
"""Generate the PROJECT-MAP wiki page: the facts about this repo that go stale.

Versions, the plugin inventory, every Alembic chain head, the ADR index and the
endpoint count were all written by hand, in several places. Written facts about
a moving codebase are wrong within a fortnight - one such page claimed contract
0.16.0 and 1159 tests while the code was at 0.18.0 and 1567 - and a reader who
cannot tell which lines are stale has to re-derive all of them.

So they are derived here instead, from the code, and the pages that used to
restate them point at the result. Anything in this file that cannot be read from
the repository does not belong in it.

Usage:
    venv/bin/python scripts/gen_project_map.py            # write the PROJECT-MAP wiki page
    venv/bin/python scripts/gen_project_map.py --check    # fail if it is out of date
    venv/bin/python scripts/gen_project_map.py --stdout   # print, write nothing

--check is the CI form: it regenerates in memory and compares, so a plugin added
without regenerating the map fails the build instead of quietly aging.
"""

import argparse
import json
import re
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
OUTPUT = REPO / 'docs' / 'PROJECT-MAP.md'

# The docmeta block is part of the generated output, not something a human
# re-adds afterwards. This script rewrites the page whole, so anything hand-added
# above the GENERATED comment survives exactly until the next regeneration - and
# tests/test_docs_index.py requires the block to be the FIRST thing in the file,
# so losing it turns a routine regeneration into a red docs gate. Emitting it
# here is the only arrangement where "regenerate" and "the gate is green" are
# the same action. Keep the summary under the 120-character limit that test
# enforces, and keep the field vocabulary in step with it.
HEADER = """<!-- docmeta
purpose: reference
audience: developer, adopter
status: current
summary: Generated inventory - versions, plugins, Alembic heads, ADR index and codebase size.
loadwith: none
-->
<!-- GENERATED by scripts/gen_project_map.py - do not edit by hand.
     Regenerate with: venv/bin/python scripts/gen_project_map.py -->

# Project map

Derived from the code on every regeneration. If something here is wrong, the
code changed and the map was not regenerated - fix it by running the generator,
never by editing this file.
"""


def _read(path):
    return path.read_text(encoding='utf-8', errors='replace')


def versions():
    """Product and plugin-contract versions, which are separate series (ADR-007,
    ADR-002) and are routinely quoted as if they were one."""
    text = _read(REPO / 'shopdb' / '__init__.py')
    found = {}
    for name in ('__version__', '__contract_version__'):
        match = re.search(r"^%s\s*=\s*['\"]([^'\"]+)['\"]" % name, text, re.M)
        found[name] = match.group(1) if match else '?'
    return found


def chainhead(versionsdir):
    """Head revision of an Alembic chain: the revision nothing else revises.

    Read from the files rather than from a database, so the map is generatable
    on a checkout with no MySQL and cannot report what one particular server
    happens to have applied.
    """
    if not versionsdir.is_dir():
        return None
    revisions, parents = set(), set()
    for path in sorted(versionsdir.glob('*.py')):
        text = _read(path)
        rev = re.search(r"^revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
        down = re.search(r"^down_revision\s*=\s*['\"]([^'\"]+)['\"]", text, re.M)
        if rev:
            revisions.add(rev.group(1))
        if down:
            parents.add(down.group(1))
    heads = sorted(revisions - parents)
    if not heads:
        return None
    # More than one head means the chain has branched, which breaks `upgrade`.
    # Reporting it is the point; hiding it behind a [0] is how it stays unnoticed.
    return ' + '.join(heads)


def plugins():
    rows = []
    for manifest in sorted((REPO / 'plugins').glob('*/manifest.json')):
        try:
            data = json.loads(_read(manifest))
        except ValueError as exc:
            rows.append({'name': manifest.parent.name, 'version': 'UNREADABLE',
                         'core': str(exc)[:60], 'head': None})
            continue
        rows.append({
            'name': data.get('name', manifest.parent.name),
            'version': data.get('version', '?'),
            'core': data.get('core_version', ''),
            'head': chainhead(manifest.parent / 'migrations' / 'versions'),
        })
    return rows


def frontendonlyplugins():
    """Directories under plugins/ with no manifest. They are core frontend
    surface that always ships (applications, for one), and they look like
    missing plugins to anyone counting directories."""
    names = []
    for path in sorted((REPO / 'plugins').iterdir()):
        # Dot-directories are build scratch (plugins/.staging), not surface.
        if path.is_dir() and not (path / 'manifest.json').exists() \
                and not path.name.startswith(('__', '.')):
            names.append(path.name)
    return names


def adrs():
    rows = []
    for path in sorted((REPO / 'docs' / 'adr').glob('ADR-*.md')):
        text = _read(path)
        title = next((line.lstrip('# ').strip()
                      for line in text.splitlines() if line.startswith('# ')),
                     path.stem)
        # Both spellings are in use: '- Status: ACCEPTED' and
        # '- **Status:** ACCEPTED'. Matching only one reported every ADR as '?',
        # which reads as "nobody decided" rather than "the regex is wrong".
        status = re.search(r'^[-*\s]*(?:\*\*)?Status(?:\*\*)?\s*:?\s*(?:\*\*)?\s*(.+)$',
                           text, re.M | re.I)
        clean = re.sub(r'[*_`]', '', status.group(1)).strip() if status else '?'
        rows.append({'file': path.name, 'title': title, 'status': clean[:40]})
    return rows


def endpointcount():
    """From the generated spec, not from a live app: this script must stay
    importable without a database or an app context."""
    spec = REPO / 'docs' / 'openapi.json'
    if not spec.is_file():
        return None
    try:
        return len(json.loads(_read(spec)).get('paths', {}))
    except ValueError:
        return None


def testcount():
    """Test FUNCTIONS, counted by reading the files.

    Deliberately not `pytest --collect-only`: that needs the venv and the whole
    import graph, and a map generator that only runs in a fully installed
    environment is one that stops being run. Parametrised cases make the real
    collected number higher; this is a floor, and it is labelled as one.
    """
    total = 0
    for path in (REPO / 'tests').rglob('test_*.py'):
        total += len(re.findall(r'^\s*def test_', _read(path), re.M))
    return total


def render():
    lines = [HEADER]
    version = versions()

    lines.append('## Versions\n')
    lines.append('| series | value | governed by |')
    lines.append('|---|---|---|')
    lines.append('| product `__version__` | `%s` | ADR-007 |'
                 % version['__version__'])
    lines.append('| plugin contract `__contract_version__` | `%s` | ADR-002 |'
                 % version['__contract_version__'])
    lines.append('')
    lines.append('They move independently. A contract bump is not a release.\n')

    lines.append('## Alembic chains (ADR-008)\n')
    lines.append('Core runs with `flask db upgrade`; every plugin chain runs')
    lines.append('with `flask plugin upgrade-all`. Both are needed on a deploy.\n')
    lines.append('| chain | head |')
    lines.append('|---|---|')
    lines.append('| core | `%s` |' % (chainhead(REPO / 'migrations' / 'versions')
                                      or 'none'))
    rows = plugins()
    for row in rows:
        if row['head']:
            lines.append('| %s | `%s` |' % (row['name'], row['head']))
    lines.append('')

    lines.append('## Bundled plugins (%d)\n' % len(rows))
    lines.append('| plugin | version | core_version | owns migrations |')
    lines.append('|---|---|---|---|')
    for row in rows:
        lines.append('| %s | %s | %s | %s |' % (
            row['name'], row['version'], row['core'] or '-',
            'yes' if row['head'] else 'no'))
    lines.append('')
    extra = frontendonlyplugins()
    if extra:
        lines.append('Manifest-less directories under `plugins/` are core '
                     'frontend surface and always ship: %s.\n'
                     % ', '.join('`%s`' % name for name in extra))

    lines.append('## Architecture decisions\n')
    lines.append('| ADR | title | status |')
    lines.append('|---|---|---|')
    for row in adrs():
        lines.append('| %s | %s | %s |' % (row['file'], row['title'], row['status']))
    lines.append('')

    lines.append('## Size\n')
    endpoints = endpointcount()
    lines.append('- test functions defined: **%d** (parametrised cases collect '
                 'higher)' % testcount())
    lines.append('- documented API paths: **%s** (`docs/openapi.json`, '
                 'regenerate with `scripts/gen_openapi.py`)'
                 % ('%d' % endpoints if endpoints is not None else 'unknown'))
    lines.append('')
    return '\n'.join(lines) + '\n'


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--check', action='store_true',
                        help='exit 1 if the committed map is out of date')
    parser.add_argument('--stdout', action='store_true',
                        help='print the map instead of writing it')
    args = parser.parse_args()

    content = render()
    if args.stdout:
        sys.stdout.write(content)
        return 0
    if args.check:
        current = _read(OUTPUT) if OUTPUT.is_file() else ''
        if current != content:
            sys.stderr.write(
                'the PROJECT-MAP wiki page is out of date.\n'
                'Regenerate: venv/bin/python scripts/gen_project_map.py\n')
            return 1
        print('the PROJECT-MAP wiki page is current')
        return 0
    OUTPUT.write_text(content, encoding='utf-8')
    print('wrote %s' % OUTPUT.relative_to(REPO))
    return 0


if __name__ == '__main__':
    raise SystemExit(main())