From 762d1b7210bc9ce73b4332f3c3af53e2b5606cbb Mon Sep 17 00:00:00 2001 From: cproudlock Date: Sun, 20 Sep 2026 12:59:49 -0400 Subject: [PATCH] Make the API usable by an agent, and stop the contract going stale Someone writing a script against ShopDB had two options: read a 705 KB spec, or guess. The spec is the reference, not an index, and handing an agent the whole thing means either truncating it or spending a context window on 479 operations nobody asked about. Three additions, each feeding the next. /api/docs/operations.txt is the index: every operation on one line with its method, path, tag, auth and summary, at around 70 KB for all 486. A caller can hold the entire API at once, find the four operations it needs, and only then fetch their schemas. /api/docs/openapi.json?tag=... narrows the spec to one of the 41 tag groups. Components are kept intact deliberately: pruning the schema graph to match would leave references resolving to nothing, and a generator that fails confusingly is worse than a file that is larger than strictly necessary. tools/mcp/ is an MCP server built on both. Five tools rather than 483, because a tool per operation is unusable - clients degrade past a few dozen and models start picking the wrong one. It reads everything from the live instance at call time, so enabling a plugin adds operations it can already reach with no change here. The token is the security boundary and the README says so plainly: every operation the credential permits is reachable, so exploratory work wants a read-scoped token. The rest of this is about the contract not drifting, which is what makes any of the above trustworthy. Two CI jobs: - api-contract runs the parity gates and then regenerates the spec and fails if anything changed. Those gates already existed and already worked - they caught both routes added here, unprompted - but nothing ran them except by hand, and the committed spec had silently drifted by three operations, including two setup endpoints added the day before. - migrations-mysql is the job the header comment has been claiming for some time while not existing. It described "the REAL multi-site deploy path" and "what catches an Alembic regression on MySQL before it ships", and there was no such job. That is worse than no comment: it is the false confidence that let a migration ship which could not build a schema on a fresh MySQL database. The suite runs on SQLite, and SQLite accepts a foreign key onto a table that does not exist. CONTRIBUTING.md now says to run gen_openapi.py --refresh after changing a route, and that a route the sweep cannot classify belongs in PUBLIC_ENDPOINTS with a reason rather than hand-edited into the inventory, which the next refresh would overwrite. The new index route is declared there alongside the spec it indexes. --- .github/workflows/ci.yml | 101 +++++++++ CONTRIBUTING.md | 26 +++ docs/api-inventory.json | 33 ++- docs/openapi.json | 120 +++++++++++ shopdb/core/api/docs.py | 88 +++++++- tests/test_core/test_public_read_surface.py | 3 + tools/mcp/README.md | 86 ++++++++ tools/mcp/pyproject.toml | 16 ++ tools/mcp/shopdb_mcp/__init__.py | 8 + tools/mcp/shopdb_mcp/client.py | 191 +++++++++++++++++ tools/mcp/shopdb_mcp/server.py | 226 ++++++++++++++++++++ 11 files changed, 894 insertions(+), 4 deletions(-) create mode 100644 tools/mcp/README.md create mode 100644 tools/mcp/pyproject.toml create mode 100644 tools/mcp/shopdb_mcp/__init__.py create mode 100644 tools/mcp/shopdb_mcp/client.py create mode 100644 tools/mcp/shopdb_mcp/server.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb25e7b8..0a57325f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ # gates - proves those checks actually RAN rather than skipping # frontend - vitest + Vue build # lean-build - ADR-013: a per-site build carries only its plugins +# api-contract - the published spec matches the code that serves it # migrations-mysql - the REAL multi-site deploy path: fresh flask db upgrade # + every plugin's chain on utf8mb4 MySQL 8, idempotent on # a second run. The pytest suite only exercises SQLite @@ -53,6 +54,106 @@ jobs: - uses: actions/checkout@v5 - run: bash scripts/check-naming-and-style.sh + api-contract: + # The published API contract has to match the code that serves it. These + # gates catch a route that exists and is undocumented, a query parameter the + # handler reads that the spec omits, and a public route nobody declared + # public. They already work; until now nothing ran them except by hand, and + # the spec had silently drifted by three operations. + runs-on: arc-runner-set + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: pip + - run: pip install -r requirements-dev.txt + - run: | + python -m pytest -q \ + tests/test_api_inventory_parity.py \ + tests/test_openapi_spec.py \ + tests/test_openapi_examples.py \ + tests/test_core/test_public_read_surface.py + - name: The spec must be regenerated, not just passing + # --refresh is derived from the code, so if it changes anything the + # committed spec was stale. Fail here rather than let a generated client + # be built from a file that no longer describes the API. + run: | + python scripts/gen_openapi.py --refresh + if ! git diff --quiet docs/openapi.json docs/api-inventory.json; then + echo "docs/openapi.json is stale. Run:" + echo " python scripts/gen_openapi.py --refresh" + git --no-pager diff --stat docs/openapi.json docs/api-inventory.json + exit 1 + fi + + migrations-mysql: + # THE JOB THE HEADER PROMISED AND NOBODY WROTE. Until now this existed only + # as a comment claiming to cover "the REAL multi-site deploy path", which is + # worse than no comment: it is exactly the false confidence that let a + # migration ship which could not build a schema on a fresh MySQL database. + # + # The pytest suite runs on SQLite, and SQLite accepts a foreign key onto a + # table that does not exist. MySQL does not. That difference is invisible to + # every other job here. + runs-on: arc-runner-set + services: + mysql: + image: mysql:8.4 + env: + MYSQL_ROOT_PASSWORD: ci + MYSQL_DATABASE: shopdb_flask + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1 -u root -pci" + --health-interval=10s --health-timeout=5s --health-retries=10 + ports: + - 3306:3306 + env: + DATABASE_URL: mysql+pymysql://root:ci@127.0.0.1:3306/shopdb_flask?charset=utf8mb4 + FLASK_APP: shopdb + SECRET_KEY: ci-not-a-secret + JWT_SECRET_KEY: ci-not-a-secret + CORS_ORIGINS: http://localhost + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: pip + - run: pip install -r requirements.txt + - name: Fresh install, exactly as the README tells a site to do it + run: | + flask db upgrade + flask plugin upgrade-all + flask seed permissions + flask seed settings + flask seed reference-data + - name: Enable every bundled plugin and migrate it + # A plugin's chain only runs once the plugin is ENABLED, so a core + # migration that depends on a plugin-owned table passes an empty install + # and fails a real one. Enabling them here is what reproduces that. + run: | + python - <<'PY' + import json, os, pathlib, datetime + root = pathlib.Path('plugins') + names = sorted(d.name for d in root.iterdir() + if (d / 'manifest.json').is_file()) + now = datetime.datetime.now().isoformat() + state = {'plugins': {n: {'name': n, 'version': '1.0.0', + 'installed_at': now, 'enabled': True, + 'migrations_applied': [], 'config': {}} + for n in names}} + out = pathlib.Path('instance'); out.mkdir(exist_ok=True) + (out / 'plugins.json').write_text(json.dumps(state, indent=2)) + print('enabled', len(names), 'plugins') + PY + flask plugin upgrade-all + - name: Re-run to prove it is idempotent + run: | + flask db upgrade + flask plugin upgrade-all + flask seed permissions + gates: # Prove the gates RAN. Several skip themselves when the file they read is # absent, which is correct in this checkout (docs/ becomes the wiki) and was diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83a739fe..b57f0f2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -169,6 +169,32 @@ Tests are required for: - New plugin hooks (contract test) - Schema migrations (before/after data integrity) +## Adding or changing an API route + +The published contract is generated from the code, so it goes stale the moment +a route changes and nobody regenerates it. A stale spec is worse than no spec: +it is what a client generator, an agent, or the MCP server in `tools/mcp/` will +believe. + +After adding or changing a route: + + python scripts/gen_openapi.py --refresh + +That rewrites `docs/api-inventory.json` and `docs/openapi.json` from the live +app. It fills in what is derived and leaves prose you have written alone, so run +it freely and then replace any seeded summary with the reason the route exists. + +If it prints `review` against your route, it could not find a gate on the view. +Do not hand-edit the inventory - the next refresh overwrites it. Either gate the +view, or, if the route is genuinely public, name it in `PUBLIC_ENDPOINTS` in +`tests/test_core/test_public_read_surface.py` with the reason. The generator +reads that list, so the published contract and the enforced gate cannot +disagree. + +The `api-contract` CI job runs the parity gates and then regenerates and fails +if anything changed, so a stale spec fails the pull request rather than being +discovered months later. + ## Branches: what belongs on main, and what does not `main` is the curated product other sites adopt, and what the publication diff --git a/docs/api-inventory.json b/docs/api-inventory.json index 2eac25b1..d429036d 100644 --- a/docs/api-inventory.json +++ b/docs/api-inventory.json @@ -2445,7 +2445,22 @@ "auth": "none (reviewed: named in the public-read or mutation sweep)", "params": "not documented", "purpose": "The generated OpenAPI 3.1 spec (machine + LLM readable).", - "example": "curl http://localhost:5001/api/docs/openapi.json" + "example": "curl http://localhost:5001/api/docs/openapi.json", + "queryparams": [ + { + "name": "tag", + "type": "string", + "required": false + } + ] + }, + { + "method": "GET", + "path": "/api/docs/operations.txt", + "auth": "none (reviewed: named in the public-read or mutation sweep)", + "params": "not documented", + "purpose": "Every operation, one line each: method, path, tag, auth, summary.", + "example": "curl http://localhost:5001/api/docs/operations.txt" }, { "method": "GET", @@ -4780,6 +4795,14 @@ { "surface": "core-setup", "endpoints": [ + { + "method": "GET", + "path": "/api/setup/checklist", + "auth": "jwt + role:admin", + "params": "not documented", + "purpose": "What is configured and what is not, with a link to each fix.", + "example": "curl http://localhost:5001/api/setup/checklist -H \"Authorization: Bearer $TOK\"" + }, { "method": "POST", "path": "/api/setup/complete", @@ -4797,6 +4820,14 @@ "example": "curl -X POST http://localhost:5001/api/setup/create-admin -H 'Content-Type: application/json' -d '{}'", "readsbody": true }, + { + "method": "GET", + "path": "/api/setup/health", + "auth": "jwt + role:admin", + "params": "not documented", + "purpose": "Post-setup reality check, including whether a restart is outstanding.", + "example": "curl http://localhost:5001/api/setup/health -H \"Authorization: Bearer $TOK\"" + }, { "method": "GET", "path": "/api/setup/needs-admin", diff --git a/docs/openapi.json b/docs/openapi.json index f5537a7c..8cb03190 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -6574,6 +6574,46 @@ "summary": "The generated OpenAPI 3.1 spec (machine + LLM readable).", "description": "The generated OpenAPI 3.1 spec (machine + LLM readable).\n\n**Auth:** none (reviewed: named in the public-read or mutation sweep)\n\n**Params:** not documented\n\n**Example:**\n```\ncurl http://localhost:5001/api/docs/openapi.json\n```", "security": [], + "responses": { + "200": { + "description": "Success. Body is the success_response envelope: {status, data, meta}.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope" + } + } + } + }, + "default": { + "description": "Error. Body is the error envelope; the code and message are nested under data.error.", + "content": { + "application/json": { + "$ref": "#/components/schemas/ErrorEnvelope" + } + } + } + }, + "parameters": [ + { + "name": "tag", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/docs/operations.txt": { + "get": { + "tags": [ + "core-docs" + ], + "summary": "Every operation, one line each: method, path, tag, auth, summary.", + "description": "Every operation, one line each: method, path, tag, auth, summary.\n\n**Auth:** none (reviewed: named in the public-read or mutation sweep)\n\n**Params:** not documented\n\n**Example:**\n```\ncurl http://localhost:5001/api/docs/operations.txt\n```", + "security": [], "responses": { "200": { "description": "Success. Body is the success_response envelope: {status, data, meta}.", @@ -12774,6 +12814,46 @@ } } }, + "/api/setup/checklist": { + "get": { + "tags": [ + "core-setup" + ], + "summary": "What is configured and what is not, with a link to each fix.", + "description": "What is configured and what is not, with a link to each fix.\n\n**Auth:** jwt + role:admin\n\n**Params:** not documented\n\n**Example:**\n```\ncurl http://localhost:5001/api/setup/checklist -H \"Authorization: Bearer $TOK\"\n```", + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "Success. Body is the success_response envelope: {status, data, meta}.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope" + } + } + } + }, + "default": { + "description": "Error. Body is the error envelope; the code and message are nested under data.error.", + "content": { + "application/json": { + "$ref": "#/components/schemas/ErrorEnvelope" + } + } + }, + "401": { + "description": "Missing or invalid credentials." + }, + "403": { + "description": "Authenticated, but not permitted." + } + } + } + }, "/api/setup/complete": { "post": { "tags": [ @@ -12868,6 +12948,46 @@ } } }, + "/api/setup/health": { + "get": { + "tags": [ + "core-setup" + ], + "summary": "Post-setup reality check, including whether a restart is outstanding.", + "description": "Post-setup reality check, including whether a restart is outstanding.\n\n**Auth:** jwt + role:admin\n\n**Params:** not documented\n\n**Example:**\n```\ncurl http://localhost:5001/api/setup/health -H \"Authorization: Bearer $TOK\"\n```", + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "Success. Body is the success_response envelope: {status, data, meta}.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope" + } + } + } + }, + "default": { + "description": "Error. Body is the error envelope; the code and message are nested under data.error.", + "content": { + "application/json": { + "$ref": "#/components/schemas/ErrorEnvelope" + } + } + }, + "401": { + "description": "Missing or invalid credentials." + }, + "403": { + "description": "Authenticated, but not permitted." + } + } + } + }, "/api/setup/needs-admin": { "get": { "tags": [ diff --git a/shopdb/core/api/docs.py b/shopdb/core/api/docs.py index f2a5be59..c975765a 100644 --- a/shopdb/core/api/docs.py +++ b/shopdb/core/api/docs.py @@ -297,12 +297,94 @@ def _spec_with_server(): @docs_bp.route('/openapi.json') def openapi_spec(): - """The generated OpenAPI 3.1 spec (machine + LLM readable).""" + """The generated OpenAPI 3.1 spec (machine + LLM readable). + + ?tag=core-assets narrows it to one tag group. The whole spec is ~700 KB + across 483 operations, which is more than a caller generating a client for + one area needs and more than most model context windows should spend. The + filtered copy keeps components intact, because a pruned schema graph is + worse than a large one: a $ref that resolves to nothing turns a working + generator into a confusing failure. + """ if not os.path.isfile(_SPEC): return Response('{"error":"openapi.json not generated"}', status=404, mimetype='application/json') - return Response(json.dumps(_spec_with_server()), - mimetype='application/json') + spec = _spec_with_server() + wanted = (request.args.get('tag') or '').strip() + if wanted: + paths = {} + for path, methods in (spec.get('paths') or {}).items(): + kept = {m: op for m, op in methods.items() + if not isinstance(op, dict) or wanted in (op.get('tags') or [])} + # A path whose only survivors are shared keys (parameters, summary) + # carries no operation and would read as an endpoint that exists + # and does nothing. + if any(isinstance(op, dict) and op.get('tags') for op in kept.values()): + paths[path] = kept + if not paths: + return Response( + json.dumps({'error': f'no operations tagged {wanted!r}', + 'tags': sorted(_operation_tags(spec))}), + status=404, mimetype='application/json') + spec = dict(spec, paths=paths) + return Response(json.dumps(spec), mimetype='application/json') + + +def _operation_tags(spec): + """Every tag that actually carries an operation.""" + tags = set() + for methods in (spec.get('paths') or {}).values(): + for op in methods.values(): + if isinstance(op, dict): + tags.update(op.get('tags') or []) + return tags + + +@docs_bp.route('/operations.txt') +def operations_index(): + """Every operation, one line each: method, path, tag, auth, summary. + + WHY THIS EXISTS. The spec is the reference and it is ~700 KB; this is the + index, and it is ~40 KB. A caller - a person or an agent writing a script - + can hold every one of the 483 operations at once here, find the four it + needs, and only then fetch their schemas from openapi.json?tag=... Handing + over the whole spec instead means either truncating it or spending a context + window on 479 operations nobody asked about. + + Plain text on purpose: it is meant to be read and grepped, not parsed. + """ + if not os.path.isfile(_SPEC): + return Response('openapi.json not generated', status=404, + mimetype='text/plain') + spec = _spec_with_server() + servers = spec.get('servers') or [{'url': '/'}] + rows = [] + for path, methods in sorted((spec.get('paths') or {}).items()): + for method, op in sorted(methods.items()): + if not isinstance(op, dict) or method.upper() not in _HTTP_METHODS: + continue + tag = (op.get('tags') or ['-'])[0] + auth = 'public' if op.get('security') == [] else ( + ','.join(sorted({k for s in (op.get('security') or []) + for k in s})) or 'bearer') + summary = ' '.join((op.get('summary') or '').split()) + rows.append(f'{method.upper():<6} {path:<52} [{tag}] ({auth}) {summary}') + header = ( + '# ShopDB API - operation index\n' + f'# {len(rows)} operations. Base URL: {servers[0].get("url", "/")}\n' + '#\n' + '# This is the INDEX. Schemas, parameters and examples are in the spec:\n' + '# /api/docs/openapi.json full (large)\n' + '# /api/docs/openapi.json?tag= one group\n' + '# Orientation for an agent, including the response envelope and auth:\n' + '# /api/docs/llms.txt\n' + '#\n' + '# METHOD path [tag] (auth) summary\n' + ) + return Response(header + '\n'.join(rows) + '\n', mimetype='text/plain') + + +_HTTP_METHODS = {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} @docs_bp.route('/redoc.standalone.js') diff --git a/tests/test_core/test_public_read_surface.py b/tests/test_core/test_public_read_surface.py index bf21321e..2fe884c5 100644 --- a/tests/test_core/test_public_read_surface.py +++ b/tests/test_core/test_public_read_surface.py @@ -58,6 +58,9 @@ PUBLIC_ENDPOINTS = { # API documentation: shapes, not data. The docs page loads its own script. 'docs.docs_index', 'docs.openapi_spec', 'docs.redoc_js', 'docs.llms_txt', + # The operation index is the same shape-not-data as the spec it indexes: + # method, path, tag and summary for routes the spec already publishes. + 'docs.operations_index', # Collector ingest, authenticated by the per-plugin collector key. 'collector.collector_schemas', 'collector.collector_status', diff --git a/tools/mcp/README.md b/tools/mcp/README.md new file mode 100644 index 00000000..a9f97b29 --- /dev/null +++ b/tools/mcp/README.md @@ -0,0 +1,86 @@ +# ShopDB MCP server + +Exposes a ShopDB instance to an LLM client (Claude Code, Claude Desktop, or any +other MCP client) so it can discover and call the API directly, instead of +someone generating a client by hand from the spec. + +## The token is the security boundary + +**Read this before pointing it at production.** + +Every operation the credential is allowed to perform is reachable through this +server, including writes and deletes. There is no separate allow-list here: the +scope of the personal access token decides what can happen, and nothing else +does. + +That is a deliberate choice, and it puts the decision where it can be enforced +server-side rather than in a client that a model is talking to. It also means: + +- For anything exploratory, create a **read-scoped token**. A read-scoped token + cannot be talked into a write, whatever the model decides to call. +- Give it the narrowest scopes that cover the task, not your admin session. +- `DELETE` against an asset database is irreversible. The tool descriptions say + so, which makes a model markedly more careful, but a token that permits it is + the only thing standing in the way. + +Create a token under Settings > API Tokens. `shopdb_whoami` reports what the +token can do; call it first. + +## Install + + pip install -e tools/mcp + +Then register it with your client. For Claude Code: + + claude mcp add shopdb \ + --env SHOPDB_URL=https://shopdb.example.net/shopdb \ + --env SHOPDB_TOKEN= \ + -- shopdb-mcp + +`SHOPDB_URL` must include the mount path if the site is served under one +(`/shopdb`), because that is what the instance's own spec declares. + +| Variable | Required | Meaning | +|---|---|---| +| `SHOPDB_URL` | yes | Base URL including any mount path | +| `SHOPDB_TOKEN` | yes | Personal access token | +| `SHOPDB_VERIFY_TLS` | no | `0` to accept a self-signed certificate. Defaults to verifying | + +## Tools + +| Tool | Purpose | +|---|---| +| `shopdb_whoami` | Instance, identity, scopes, operation count. Call first | +| `shopdb_search_operations` | Find operations by keyword or tag. Start here | +| `shopdb_describe_operation` | Parameters, request body and responses for one operation | +| `shopdb_call` | Call any operation | +| `shopdb_llms_guide` | The instance's own orientation guide | + +## Why five tools and not 483 + +ShopDB publishes 483 operations. One tool each would be unusable: MCP clients +degrade badly past a few dozen tools and models start choosing wrong ones. So +the surface is small and generic, and the instance's own published index is what +makes it navigable - `search` reads `/api/docs/operations.txt` (about 70 KB for +every operation) and `describe` fetches only the tag group it needs rather than +the 700 KB spec. + +Nothing is generated ahead of time. Enabling a plugin adds operations this +server can already reach, with no change here and no reinstall. + +## Staying current + +The server reads everything from the live instance, so it describes whatever +that instance actually serves. The staleness risk is upstream: `docs/openapi.json` +is generated and committed, and if it drifts, the instance serves a stale spec +and this server faithfully reports stale information. + +That is guarded by the `api-contract` CI job and by +`python scripts/gen_openapi.py --refresh`. See CONTRIBUTING.md. + +## Limitations + +- Air-gapped sites need `mcp` vendored; it is not bundled here. +- No streaming or subscriptions - every tool is a request and a response. +- Responses over 100 KB are truncated with a note. Paginate instead + (`perpage` is silently capped at 100 by the API). diff --git a/tools/mcp/pyproject.toml b/tools/mcp/pyproject.toml new file mode 100644 index 00000000..5f905d3e --- /dev/null +++ b/tools/mcp/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "shopdb-mcp" +version = "0.1.0" +description = "MCP server for a ShopDB instance" +requires-python = ">=3.10" +dependencies = ["mcp>=1.2"] + +[project.scripts] +shopdb-mcp = "shopdb_mcp:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["shopdb_mcp*"] diff --git a/tools/mcp/shopdb_mcp/__init__.py b/tools/mcp/shopdb_mcp/__init__.py new file mode 100644 index 00000000..efface43 --- /dev/null +++ b/tools/mcp/shopdb_mcp/__init__.py @@ -0,0 +1,8 @@ +"""MCP server exposing a ShopDB instance to an LLM client.""" + +__all__ = ['main'] + + +def main(): + from .server import main as _main + _main() diff --git a/tools/mcp/shopdb_mcp/client.py b/tools/mcp/shopdb_mcp/client.py new file mode 100644 index 00000000..f3264cba --- /dev/null +++ b/tools/mcp/shopdb_mcp/client.py @@ -0,0 +1,191 @@ +"""HTTP access to a ShopDB instance, and the operation index it publishes. + +Everything the server knows about the API is fetched from the instance at +runtime: the operation index, the schemas, the mount path. Nothing is generated +ahead of time and nothing is pinned, so a site running an older or newer build +is described accurately rather than according to whatever was true when this +package was written. +""" + +from __future__ import annotations + +import json +import os +import re +import threading +import time +from dataclasses import dataclass +from typing import Any +from urllib import error, parse, request + +# How long a fetched index or spec stays usable before it is re-read. Long +# enough that a burst of tool calls does not re-download a 700 KB document, +# short enough that enabling a plugin shows up without restarting the server. +CACHE_SECONDS = 300 + +DEFAULT_TIMEOUT = 30 + +# Verbs that change or destroy data. Named so the tool descriptions can say so: +# a model told that DELETE is irreversible behaves better than one left to infer +# it from the verb. +WRITE_METHODS = {'POST', 'PUT', 'PATCH', 'DELETE'} +DESTRUCTIVE_METHODS = {'DELETE'} + + +class ShopdbError(RuntimeError): + """An error worth showing the caller verbatim.""" + + +@dataclass +class Operation: + method: str + path: str + tag: str + auth: str + summary: str + + def matches(self, needle: str) -> bool: + blob = f'{self.method} {self.path} {self.tag} {self.summary}'.lower() + return all(word in blob for word in needle.lower().split()) + + +class ShopdbClient: + """Talks to one ShopDB instance with one credential.""" + + def __init__(self, base_url: str | None = None, token: str | None = None, + verify_tls: bool | None = None, timeout: int = DEFAULT_TIMEOUT): + self.base_url = (base_url or os.environ.get('SHOPDB_URL') or '').rstrip('/') + self.token = token or os.environ.get('SHOPDB_TOKEN') or '' + if verify_tls is None: + # Sites commonly run a self-signed certificate on an internal name, + # so this has to be settable - but it defaults to verifying, and the + # caller has to say otherwise deliberately. + verify_tls = (os.environ.get('SHOPDB_VERIFY_TLS', '1').strip().lower() + not in ('0', 'false', 'no')) + self.verify_tls = verify_tls + self.timeout = timeout + self._lock = threading.Lock() + self._cache: dict[str, tuple[float, Any]] = {} + + if not self.base_url: + raise ShopdbError( + 'SHOPDB_URL is not set. Point it at the instance, including any ' + 'mount path, for example https://shopdb.example.net/shopdb') + + # ---- plumbing -------------------------------------------------------- + + def _opener(self): + if self.verify_tls: + return request.build_opener() + import ssl + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return request.build_opener(request.HTTPSHandler(context=ctx)) + + def request(self, method: str, path: str, params: dict | None = None, + body: Any = None) -> tuple[int, Any]: + """One HTTP call. Returns (status, parsed-body-or-text).""" + url = self.base_url + '/' + path.lstrip('/') + if params: + # Drop empty values rather than sending key= , which several list + # endpoints treat as a filter on the empty string. + clean = {k: v for k, v in params.items() if v not in (None, '')} + if clean: + url += ('&' if '?' in url else '?') + parse.urlencode(clean, doseq=True) + + data = None + headers = {'Accept': 'application/json'} + if self.token: + headers['Authorization'] = f'Bearer {self.token}' + if body is not None: + data = json.dumps(body).encode('utf-8') + headers['Content-Type'] = 'application/json' + + req = request.Request(url, data=data, headers=headers, method=method.upper()) + try: + with self._opener().open(req, timeout=self.timeout) as response: + return response.status, _decode(response.read(), + response.headers.get('Content-Type')) + except error.HTTPError as exc: + # An error body from this API is the useful part (it carries the + # error code and message), so return it rather than raising. + return exc.code, _decode(exc.read(), exc.headers.get('Content-Type')) + except error.URLError as exc: + raise ShopdbError(f'could not reach {self.base_url}: {exc.reason}') from exc + + def _cached(self, key: str, fetch): + with self._lock: + hit = self._cache.get(key) + if hit and (time.time() - hit[0]) < CACHE_SECONDS: + return hit[1] + value = fetch() + with self._lock: + self._cache[key] = (time.time(), value) + return value + + # ---- what the instance publishes about itself ------------------------ + + def operations(self) -> list[Operation]: + """The published operation index, parsed. + + Read from /api/docs/operations.txt rather than the full spec: the index + is around 70 KB against 700 KB, and listing what exists does not need + the schemas. + """ + def fetch(): + status, text = self.request('GET', '/api/docs/operations.txt') + if status != 200: + raise ShopdbError( + f'operation index unavailable (HTTP {status}). This instance ' + 'may predate /api/docs/operations.txt.') + return _parse_index(text if isinstance(text, str) else str(text)) + return self._cached('operations', fetch) + + def spec(self, tag: str | None = None) -> dict: + """The OpenAPI spec, optionally narrowed to one tag group.""" + def fetch(): + path = '/api/docs/openapi.json' + status, payload = self.request('GET', path, params={'tag': tag} if tag else None) + if status != 200: + raise ShopdbError(f'spec unavailable (HTTP {status}): {payload}') + return payload + return self._cached(f'spec:{tag or "*"}', fetch) + + def find_operation(self, method: str, path: str) -> dict | None: + """The spec entry for one operation, fetched by its tag to stay small.""" + method = method.lower() + for op in self.operations(): + if op.method.lower() == method and op.path == path: + spec = self.spec(op.tag) + return (spec.get('paths', {}).get(path, {}) or {}).get(method) + return None + + +_INDEX_LINE = re.compile( + r'^(?P[A-Z]+)\s+(?P\S+)\s+\[(?P[^\]]*)\]\s+' + r'\((?P[^)]*)\)\s*(?P.*)$') + + +def _parse_index(text: str) -> list[Operation]: + out = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + m = _INDEX_LINE.match(line) + if m: + out.append(Operation(m.group('method'), m.group('path'), + m.group('tag'), m.group('auth'), + m.group('summary').strip())) + return out + + +def _decode(raw: bytes, content_type: str | None): + text = raw.decode('utf-8', errors='replace') + if content_type and 'json' in content_type.lower(): + try: + return json.loads(text) + except json.JSONDecodeError: + return text + return text diff --git a/tools/mcp/shopdb_mcp/server.py b/tools/mcp/shopdb_mcp/server.py new file mode 100644 index 00000000..56041dc8 --- /dev/null +++ b/tools/mcp/shopdb_mcp/server.py @@ -0,0 +1,226 @@ +"""MCP server for a ShopDB instance. + +WHY IT IS SHAPED LIKE THIS. ShopDB publishes 483 operations. Exposing one tool +per operation would be useless: MCP clients degrade badly past a few dozen tools +and a model starts picking the wrong one. So the surface here is small and +generic - find an operation, read its contract, call it - and the instance's own +published index is what makes that navigable. Adding a plugin adds operations +that this server can already reach, with no change here. + +SECURITY MODEL: the token IS the boundary. Every operation the credential is +allowed to perform is reachable, including writes and deletes. That is a +deliberate choice, and it means the scope of the personal access token decides +the blast radius, not this code. Create a read-scoped token for anything +exploratory. See README.md. +""" + +from __future__ import annotations + +import json +import os +from typing import Any + +# The SDK renamed FastMCP to MCPServer in 2.0. The two have the same shape for +# what this server uses (a .tool() decorator and .run()), so support both rather +# than forcing a pin: a site may already have either installed. +try: + from mcp.server.mcpserver import MCPServer as _Server # mcp >= 2 +except ImportError: # pragma: no cover + from mcp.server.fastmcp import FastMCP as _Server # mcp 1.x + +from .client import (DESTRUCTIVE_METHODS, WRITE_METHODS, ShopdbClient, + ShopdbError) + +mcp = _Server('shopdb') + +_client: ShopdbClient | None = None + + +def client() -> ShopdbClient: + global _client + if _client is None: + _client = ShopdbClient() + return _client + + +def _fail(message: str) -> str: + """Errors come back as text the model can read and act on. + + Raising would surface a stack trace the model cannot do anything with; a + sentence telling it what went wrong and what to try next is more useful. + """ + return f'ERROR: {message}' + + +@mcp.tool() +def shopdb_whoami() -> str: + """Check which ShopDB instance this server targets and what the token can do. + + Call this first. It confirms the instance is reachable, the credential is + valid, and - importantly - what the token is scoped to, which determines + which other calls will succeed. + """ + try: + c = client() + except ShopdbError as exc: + return _fail(str(exc)) + try: + status, body = c.request('GET', '/api/auth/me') + except ShopdbError as exc: + return _fail(str(exc)) + if status == 401: + return _fail('token rejected (HTTP 401). Check SHOPDB_TOKEN.') + if status != 200: + return _fail(f'HTTP {status} from /api/auth/me: {body}') + user = (body or {}).get('data') or {} + try: + count = len(c.operations()) + except ShopdbError as exc: + count = f'unknown ({exc})' + return json.dumps({ + 'instance': c.base_url, + 'username': user.get('username'), + 'roles': user.get('roles'), + 'permissions': len(user.get('permissions') or []), + 'operations_published': count, + 'tls_verified': c.verify_tls, + }, indent=2) + + +@mcp.tool() +def shopdb_search_operations(query: str = '', tag: str = '', + limit: int = 40) -> str: + """Find API operations by keyword. START HERE to discover what exists. + + Searches method, path, tag and summary. `query` words are ANDed, so + "printer supplies" matches an operation mentioning both. `tag` narrows to + one group (for example plugin-printers, core-assets). + + Returns one line per operation. Use shopdb_describe_operation for the + parameters and body schema of anything you intend to call. + """ + try: + ops = client().operations() + except ShopdbError as exc: + return _fail(str(exc)) + if tag: + ops = [o for o in ops if o.tag == tag] + if query: + ops = [o for o in ops if o.matches(query)] + if not ops: + tags = sorted({o.tag for o in client().operations()}) + return (f'No operations matched query={query!r} tag={tag!r}.\n' + f'Available tags: {", ".join(tags)}') + lines = [f'{o.method} {o.path} [{o.tag}] ({o.auth}) {o.summary}' + for o in ops[:limit]] + more = len(ops) - len(lines) + if more > 0: + lines.append(f'... and {more} more; narrow the query or raise limit') + return '\n'.join(lines) + + +@mcp.tool() +def shopdb_describe_operation(method: str, path: str) -> str: + """Full contract for one operation: parameters, request body, responses. + + `path` must be the templated form exactly as shopdb_search_operations + printed it, for example /api/machines/{machineid} - not a filled-in URL. + + Read this before calling anything you have not called before. It also warns + when the operation writes or deletes. + """ + try: + op = client().find_operation(method, path) + except ShopdbError as exc: + return _fail(str(exc)) + if op is None: + return _fail(f'no operation {method.upper()} {path}. ' + 'Use shopdb_search_operations to find the exact path.') + out = { + 'method': method.upper(), + 'path': path, + 'summary': op.get('summary'), + 'description': op.get('description'), + 'parameters': op.get('parameters'), + 'requestBody': op.get('requestBody'), + 'responses': {k: {'description': v.get('description')} + for k, v in (op.get('responses') or {}).items()}, + } + warning = '' + if method.upper() in DESTRUCTIVE_METHODS: + warning = ('\n\nWARNING: this DELETES data and cannot be undone. ' + 'Confirm with the user before calling it.') + elif method.upper() in WRITE_METHODS: + warning = '\n\nNote: this MODIFIES data.' + return json.dumps(out, indent=2, default=str) + warning + + +@mcp.tool() +def shopdb_call(method: str, path: str, params: dict[str, Any] | None = None, + body: dict[str, Any] | None = None) -> str: + """Call any ShopDB operation. Use shopdb_describe_operation first. + + `path` may contain filled-in ids (/api/machines/42). `params` become the + query string; `body` is sent as JSON. + + Every response is the envelope {status, data, meta} - the payload you want + is under "data", not at the top level. List endpoints paginate with page and + perpage, and perpage is silently capped at 100, so asking for 500 returns + 100 without an error. + + This can write and delete if the token allows it. For DELETE, confirm with + the user first. + """ + try: + c = client() + except ShopdbError as exc: + return _fail(str(exc)) + verb = method.upper() + try: + status, payload = c.request(verb, path, params=params, body=body) + except ShopdbError as exc: + return _fail(str(exc)) + + if status == 401: + return _fail('HTTP 401. The token is missing, expired or rejected.') + if status == 403: + return _fail('HTTP 403. The token lacks the scope or role for this ' + 'operation - check shopdb_whoami.') + if status == 404 and isinstance(payload, str): + return _fail(f'HTTP 404 and a non-JSON body, which usually means the ' + f'path is wrong rather than the record missing: {path}') + text = payload if isinstance(payload, str) else json.dumps(payload, indent=2, + default=str) + if len(text) > 100_000: + text = text[:100_000] + '\n... truncated; narrow with params or paginate' + return f'HTTP {status}\n{text}' + + +@mcp.tool() +def shopdb_llms_guide() -> str: + """The instance's own orientation guide for agents. + + Short. Covers the response envelope, authentication and how the rest of the + documentation is laid out. Worth reading once at the start of a session. + """ + try: + status, text = client().request('GET', '/api/docs/llms.txt') + except ShopdbError as exc: + return _fail(str(exc)) + if status != 200: + return _fail(f'guide unavailable (HTTP {status})') + return text if isinstance(text, str) else str(text) + + +def main() -> None: + if not os.environ.get('SHOPDB_URL'): + raise SystemExit( + 'SHOPDB_URL is not set.\n' + ' SHOPDB_URL=https://shopdb.example.net/shopdb\n' + ' SHOPDB_TOKEN=\n' + 'See tools/mcp/README.md.') + mcp.run() + + +if __name__ == '__main__': + main()