shopdb-flask/tests/test_docs_index.py

"""docs/INDEX.md is the router, and a router nobody checks is a router that lies.

The corpus is larger than any reader - person or agent - can hold, so the only
thing that makes it usable is being able to find the minimum correct set of
pages for a task and skip the rest. That needs two things to stay true, and
neither survives on good intentions:

  1. Every page declares what it is for and whether it is still current, in a
     `docmeta` block a machine can filter on without opening the page.
  2. docs/INDEX.md routes every current page, and quarantines the ones that are
     superseded or are only a record.

The failure this gate exists to stop is the quiet one: a page is added, nothing
routes to it, and it rots unread - or worse, a page goes `historical` and keeps
being followed because it is still sitting in a task row.

A second pass over the whole corpus added three more, because the first three
turned out to be satisfiable by an index that was still useless:

  3. Being NAMED somewhere in INDEX.md is not the same as being routed. A
     current page that appears only in a paragraph, or only in the Archive
     section's prose, is a page no task row sends anyone to. It must be in a
     row.
  4. The line counts have to be right, not roughly right. The cost column is
     the reason to trust the index over opening pages at random, and a corpus
     that is being cut shrinks pages constantly - so a stale number is the
     normal failure, not an exotic one.
  5. The two routers - INDEX.md and START-HERE.md - must not point at a file
     that is not there. Pages get merged and renamed; a dead link in the router
     is worse than a missing row, because the reader trusts it.

Line-count tolerance: INDEX.md prints what a row costs to read, so a reader can
see the price before paying it. That number is checked against the real files
within 3 percent or 10 lines, whichever is larger. That absorbs a reflowed
paragraph and nothing else: the tolerance used to be 10 percent or 25 lines,
which let a 52-line row claim 77 and let whole sections drift after an edit
pass. If a row is out by more than a reflow, the number is simply wrong and
recomputing it is one command.
"""

import re
from pathlib import Path

import pytest

REPO = Path(__file__).resolve().parents[1]
DOCS = REPO / 'docs'
INDEX = DOCS / 'INDEX.md'
START_HERE = DOCS / 'START-HERE.md'

# The two routers. Everything else in docs/ is a destination; these two are how
# a reader arrives, so they are exempt from "must sit in a task row" (a row
# pointing at the index the row is in would be noise) and their links are held
# to a stricter standard than anyone else's.
ROUTERS = (INDEX, START_HERE)

# Same skip rule as tests/test_docs_publishable.py: docs/ prose is stripped from
# the published mirror and lives in the wiki, so a checkout with no docs/*.md
# has nothing to route and this module is inapplicable. docs/openapi.json is
# tracked, so keying off the DIRECTORY would make this run on the mirror.
pytestmark = pytest.mark.skipif(
    not any(DOCS.glob('*.md')),
    reason='no docs/ prose in this checkout - it is published to the wiki instead')

PURPOSES = {'tutorial', 'howto', 'reference', 'explanation'}
AUDIENCES = {'operator', 'adopter', 'developer', 'integrator'}
STATUSES = {'current', 'legacy', 'historical'}
FIELDS = ('purpose', 'audience', 'status', 'summary', 'loadwith')
ARCHIVED = {'legacy', 'historical'}

SUMMARY_LIMIT = 120

# The block must be the FIRST thing in the file, before the H1. A GitHub wiki
# renders YAML front matter as visible junk, which is why this is an HTML
# comment, and a comment further down the page is one a reader trips over.
DOCMETA = re.compile(r'\A<!-- docmeta\n(.*?)\n-->\n', re.S)
LINK = re.compile(r'\[[^\]]*\]\(([^)#]+)(?:#[^)]*)?\)')
ARCHIVE_HEADING = re.compile(r'^##\s+Archive\b', re.M)
ARCHIVE_TOTAL = re.compile(r'they are ([\d,]+) lines')
ALL_LINKS = re.compile(r'\[[^\]]*\]\(([^)]+)\)')


def pages():
    """Every authored page in docs/, including adr/ and proposals/."""
    return sorted(p for p in DOCS.rglob('*.md'))


def rel(path):
    return path.relative_to(DOCS).as_posix()


def line_count(path):
    return len(path.read_text(errors='replace').splitlines())


def parse_docmeta(path):
    """Return the docmeta fields, or None when the block is missing."""
    match = DOCMETA.match(path.read_text(errors='replace'))
    if not match:
        return None
    fields = {}
    for line in match.group(1).splitlines():
        key, sep, value = line.partition(':')
        if not sep:
            return None
        fields[key.strip()] = value.strip()
    return fields


def value(fields, key):
    """A field with the template's trailing '# comment' hint removed."""
    return re.sub(r'\s+#.*$', '', fields.get(key, '')).strip()


def status_of(path):
    fields = parse_docmeta(path)
    return value(fields, 'status') if fields else None


def links_in(text):
    """Every in-repo .md target a page links to, as written."""
    return [t for t in LINK.findall(text) if t.endswith('.md') and '://' not in t]


def all_targets_in(text):
    """Every relative link target on a page, whatever its extension.

    links_in() only sees .md, which is right for routing questions and wrong
    for "does this file exist": the index links llms.txt, and a router that
    points at a file nobody kept is the failure this catches.
    """
    out = []
    for target in ALL_LINKS.findall(text):
        target = target.split()[0] if target.split() else target
        if '://' in target or target.startswith(('mailto:', '#')):
            continue
        out.append(target.split('#')[0])
    return [t for t in out if t]


def resolve(page, target):
    """A link target on `page`, as a path under docs/, or None if it escapes."""
    candidate = (page.parent / target).resolve()
    try:
        candidate.relative_to(DOCS.resolve())
    except ValueError:
        return None
    return candidate


def index_sections():
    """INDEX.md split into (task side, archive side) at the Archive heading."""
    text = INDEX.read_text()
    match = ARCHIVE_HEADING.search(text)
    assert match, 'docs/INDEX.md must carry a "## Archive" section heading'
    return text[:match.start()], text[match.start():]


def table_rows(text):
    """(link targets, stated line total) for every table row that states one."""
    rows = []
    for line in text.splitlines():
        line = line.strip()
        if not line.startswith('|') or set(line) <= set('|- '):
            continue
        cells = [c.strip() for c in line.strip('|').split('|')]
        if len(cells) < 2 or not cells[-1].isdigit():
            continue
        targets = links_in(line)
        if targets:
            rows.append((targets, int(cells[-1])))
    return rows


def tolerance(real):
    """How far a stated line count may sit from the real one. See the docstring."""
    return max(10, real * 0.03)


def test_there_is_something_to_check():
    """A path change that matched nothing would make this whole file vacuous."""
    assert len(pages()) > 20, 'docs/ prose disappeared, or the glob stopped matching'
    assert INDEX.is_file(), 'docs/INDEX.md is missing - there is no router'
    assert len(table_rows(index_sections()[0])) > 20, 'docs/INDEX.md routes almost nothing'


@pytest.mark.parametrize('page', pages(), ids=rel)
def test_every_page_carries_a_docmeta_block(page):
    fields = parse_docmeta(page)
    assert fields is not None, (
        '%s has no docmeta block. It must be the first thing in the file, '
        'before the H1: an HTML comment opening "<!-- docmeta" and closing '
        'with "-->", one "key: value" per line.' % rel(page))
    missing = [f for f in FIELDS if f not in fields]
    assert not missing, '%s docmeta is missing: %s' % (rel(page), ', '.join(missing))
    unknown = [f for f in fields if f not in FIELDS]
    assert not unknown, '%s docmeta has unknown fields: %s' % (rel(page), ', '.join(unknown))


@pytest.mark.parametrize('page', pages(), ids=rel)
def test_docmeta_uses_the_allowed_vocabularies(page):
    fields = parse_docmeta(page)
    if fields is None:
        pytest.skip('no docmeta - reported by test_every_page_carries_a_docmeta_block')

    purpose = value(fields, 'purpose')
    assert purpose in PURPOSES, (
        '%s: purpose "%s" is not one of %s. A page that is two of these is a '
        'page that should be split.' % (rel(page), purpose, sorted(PURPOSES)))

    audience = [a.strip() for a in value(fields, 'audience').split(',') if a.strip()]
    assert audience, '%s: audience is empty' % rel(page)
    bad = [a for a in audience if a not in AUDIENCES]
    assert not bad, '%s: audience %s not in %s' % (rel(page), bad, sorted(AUDIENCES))

    status = value(fields, 'status')
    assert status in STATUSES, (
        '%s: status "%s" is not one of %s' % (rel(page), status, sorted(STATUSES)))


@pytest.mark.parametrize('page', pages(), ids=rel)
def test_summary_answers_is_this_the_page_i_need(page):
    fields = parse_docmeta(page)
    if fields is None:
        pytest.skip('no docmeta - reported by test_every_page_carries_a_docmeta_block')
    summary = fields.get('summary', '').strip()
    assert summary, '%s: summary is empty. It is what a router shows INSTEAD of the page.' % rel(page)
    assert len(summary) <= SUMMARY_LIMIT, (
        '%s: summary is %d chars, limit is %d. Say what question the page '
        'answers, not what it contains.' % (rel(page), len(summary), SUMMARY_LIMIT))


@pytest.mark.parametrize('page', pages(), ids=rel)
def test_loadwith_targets_exist(page):
    """loadwith names the pages this one is not complete without."""
    fields = parse_docmeta(page)
    if fields is None:
        pytest.skip('no docmeta - reported by test_every_page_carries_a_docmeta_block')
    raw = value(fields, 'loadwith')
    if not raw or raw.lower() == 'none':
        return
    for name in [n.strip() for n in raw.split(',') if n.strip()]:
        target = resolve(page, name) or (DOCS / name)
        if not target.is_file():
            target = DOCS / name
        assert target.is_file(), (
            '%s: loadwith names "%s", which does not exist' % (rel(page), name))


@pytest.mark.parametrize(
    'page', [p for p in pages() if p != INDEX], ids=lambda p: rel(p))
def test_every_page_is_routed_or_archived(page):
    """A page nothing routes to is a page nobody reads and nobody maintains."""
    text = INDEX.read_text()
    targets = {resolve(INDEX, t) for t in links_in(text)}
    assert page in targets, (
        '%s is in docs/ but docs/INDEX.md does not name it. Add a task row for '
        'it, or - if it is superseded or only a record - mark it legacy or '
        'historical in its docmeta and list it in the Archive section.' % rel(page))


@pytest.mark.parametrize('router', ROUTERS, ids=lambda p: rel(p))
def test_a_router_never_points_at_a_file_that_is_not_there(router):
    """INDEX.md and START-HERE.md are the two pages everyone is sent to first.

    A dead link here is worse than a missing row: a missing row makes a reader
    look further, a dead link makes them think the answer is gone. Pages get
    merged, split and renamed, and both routers name files by hand, so this is
    the check that catches it the same day.

    Every relative target counts, not just .md - the index links llms.txt too -
    and a target outside docs/ is dead by definition once these pages are
    published to the wiki, which carries docs/ and nothing above it.
    """
    broken = []
    for target in sorted(set(all_targets_in(router.read_text()))):
        page = resolve(router, target)
        if page is None:
            broken.append('%s (outside docs/)' % target)
        elif not page.is_file():
            broken.append(target)
    assert not broken, (
        'docs/%s links to files that do not exist: %s'
        % (rel(router), ', '.join(broken)))


def test_every_current_page_is_reachable_from_a_task_row():
    """Named in the index is not the same as routed by it.

    Nothing stops a page being mentioned in a paragraph, or slipped into the
    Archive section's prose, and passing "INDEX.md names it". A reader works
    the rows: they find the row that matches the job and read what it lists. A
    current page that is in no row is a page that gets written, gets stale and
    never gets opened - and the cost of keeping it is paid by everyone who
    greps the corpus.

    The routers themselves are exempt: they are where a reader starts.
    """
    tasks, _ = index_sections()
    routed = set()
    for targets, _lines in table_rows(tasks):
        for target in targets:
            page = resolve(INDEX, target)
            if page:
                routed.add(page)

    stranded = sorted(
        rel(p) for p in pages()
        if status_of(p) == 'current' and p not in routed and p not in ROUTERS)
    assert not stranded, (
        'these pages are current guidance but no task row in docs/INDEX.md '
        'sends anyone to them: %s. Give each one a row saying what task it '
        'answers, or - if it is superseded or only a record - mark it legacy '
        'or historical and move it to the Archive section.' % ', '.join(stranded))


def test_task_rows_never_route_to_an_archived_page():
    """Historical and legacy pages belong in the Archive section, nowhere else."""
    tasks, _ = index_sections()
    offenders = []
    for target in sorted(set(links_in(tasks))):
        page = resolve(INDEX, target)
        if page and page.is_file() and status_of(page) in ARCHIVED:
            offenders.append('%s (%s)' % (target, status_of(page)))
    assert not offenders, (
        'docs/INDEX.md routes a task to a page that is not current guidance: %s'
        % ', '.join(offenders))


def test_the_archive_section_holds_nothing_current():
    _, archive = index_sections()
    offenders = []
    for target in sorted(set(links_in(archive))):
        page = resolve(INDEX, target)
        if page and page.is_file() and status_of(page) == 'current':
            offenders.append(target)
    assert not offenders, (
        'docs/INDEX.md files these current pages under Archive, where readers '
        'are told not to follow them: %s' % ', '.join(offenders))


def test_no_current_page_hides_behind_an_archived_one():
    """Walk out from INDEX.md without passing through legacy or historical pages.

    A current page that can only be reached through a superseded one is a page
    a careful reader will never open, because they were told to skip the page
    that links it.
    """
    seen = set()
    queue = [INDEX]
    while queue:
        page = queue.pop()
        if page in seen:
            continue
        seen.add(page)
        if status_of(page) in ARCHIVED:
            continue  # a dead end on purpose: nothing may depend on it
        for target in links_in(page.read_text(errors='replace')):
            nxt = resolve(page, target)
            if nxt and nxt.is_file():
                queue.append(nxt)

    stranded = sorted(
        rel(p) for p in pages() if status_of(p) == 'current' and p not in seen)
    assert not stranded, (
        'these current pages are only reachable through an archived page: %s'
        % ', '.join(stranded))


def test_index_line_totals_are_honest():
    """The cost column is the point of the index. A wrong one is worse than none."""
    tasks, archive = index_sections()
    offenders = []
    for targets, stated in table_rows(tasks) + table_rows(archive):
        real = 0
        for target in targets:
            page = resolve(INDEX, target)
            if page and page.is_file():
                real += line_count(page)
        if abs(real - stated) > tolerance(real):
            offenders.append('%s: says %d lines, is %d' % ('+'.join(targets), stated, real))
    assert not offenders, (
        'docs/INDEX.md misstates what a row costs to read:\n  %s' % '\n  '.join(offenders))


def test_the_archive_total_is_honest():
    _, archive = index_sections()
    match = ARCHIVE_TOTAL.search(archive)
    assert match, 'the Archive section should say how many lines it adds up to'
    stated = int(match.group(1).replace(',', ''))
    real = sum(count for _, count in table_rows(archive))
    assert abs(real - stated) <= tolerance(real), (
        'the Archive section claims %d lines; its rows add up to %d' % (stated, real))