ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

guides export/import bundles

Browse Source
This commit is contained in:
Cameron Proudlock
2026-08-27 11:47:01 -04:00
parent 28798e6a32
commit d38a599292
4 changed files with 469 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
CHANGELOG.md
View File

@@ -22,6 +22,19 @@ ADR-007 and ADR-002.
tables; in self-hosted mode one stored loan expands into a take and a return,
so a returned drive is distinguishable from one still out.
- **`flask guides export` / `flask guides import` - a guide can reach another
site.** Sites are separate databases and the ones that matter are air-gapped
from wherever a guide gets written, so a procedure written once was stranded
where it was typed. Export writes one `.guide.json` per guide with its images
carried inside it - one file, because a bundle that needs its images copied
alongside it is a bundle that will eventually arrive without them - and
import reads them back. The slug travels, since that is the address somebody
pastes into a work order and the same guide should answer to the same name on
every site; ids, revision history and publication state do not, because those
belong to the database that issued them. Imports arrive as drafts;
`--replace` updates a guide already carrying that slug rather than creating a
second one, which is how a corrected guide is re-shipped.
- **`flask guides import-html` - an existing HTML article becomes a guide.**
A site adopting this app arrives holding procedures already written: styled
HTML pages that get mailed around and opened off a share. Retyping one into

186
plugins/guides/cli.py
View File

@@ -1,15 +1,22 @@
"""CLI: bring an existing HTML article in as a guide.
"""CLI: move guides in and out of a site.
flask guides import-html PATH [PATH ...]
flask guides import-html PATH [PATH ...] an HTML article becomes a guide
flask guides export --out DIR guides become portable files
flask guides import PATH [PATH ...] those files become guides here
A site that adopts this app usually arrives with a folder of procedures already
written - styled HTML pages that get mailed around and opened off a share. This
imports them so they live where they are used, become searchable, tickable and
printable, and stop being a file somebody has to find.
Imports land as DRAFTS, always. The conversion is faithful but it is not a
render: layout, colour and anything decorative are dropped on purpose, and a
person should read the result before anybody is told to follow it.
Imports land as DRAFTS, always - both kinds. An HTML conversion is faithful but
it is not a render, and a bundle written on another site has not been read by
anybody here. Publishing is a decision somebody makes after reading.
Sites are separate databases and the useful ones are air-gapped from where a
guide gets written, so export/import is how the same procedure reaches all of
them: one JSON file per guide, images carried inside it, moved by whatever
moves files to that site.
"""
import click
@@ -18,7 +25,7 @@ from flask.cli import AppGroup, with_appcontext
from shopdb.api import db
from .models import Guide, GuideImage
from .services import authoring
from .services import authoring, bundle as bundleservice
from .services.blocks import BlockError
from .services.htmlimport import parse
@@ -107,6 +114,117 @@ def build_cli():
click.echo(' imported as /guides/{} (draft, revision {})'
.format(guide.slug, guide.revision))
@guides_cli.command('export')
@click.option('--out', 'outdir', required=True,
type=click.Path(file_okay=False),
help='Directory to write the bundles into. Created if needed.')
@click.option('--slug', 'slugs', multiple=True,
help='Export just this guide. Repeatable. Default: all.')
@click.option('--published-only', is_flag=True,
help='Skip guides that have never been published.')
@with_appcontext
def export_guides(outdir, slugs, published_only):
"""Write guides out as portable .guide.json files."""
import os
query = Guide.query
if slugs:
query = query.filter(Guide.slug.in_(slugs))
if published_only:
query = query.filter(Guide.publishedrevisionid.isnot(None))
guides = query.order_by(Guide.slug).all()
missing = set(slugs) - {guide.slug for guide in guides}
for slug in sorted(missing):
click.echo(' NOT FOUND {}'.format(slug))
if not guides:
click.echo('Nothing to export.')
return
os.makedirs(outdir, exist_ok=True)
for guide in guides:
images = GuideImage.query.filter_by(guideid=guide.guideid).all()
payload = bundleservice.build(guide, images)
path = os.path.join(outdir, '{}.guide.json'.format(guide.slug))
with open(path, 'w', encoding='utf-8') as handle:
handle.write(bundleservice.dumps(payload))
click.echo(' {} {} blocks, {} image(s), {:.0f} KB{}'.format(
os.path.basename(path), len(payload['blocks']),
len(payload['images']), os.path.getsize(path) / 1024,
'' if guide.publishedrevisionid else ' [draft here]'))
click.echo('{} guide(s) -> {}'.format(len(guides), outdir))
@guides_cli.command('import')
@click.argument('paths', nargs=-1, required=True,
type=click.Path(exists=True))
@click.option('--userid', type=int, default=None,
help='Record this user as the author on this site.')
@click.option('--replace', is_flag=True,
help='If a guide already has the bundle\'s slug, edit it in '
'place (a new revision) instead of creating a second '
'one. This is how an updated guide is re-shipped.')
@click.option('--apply/--dry-run', default=False,
help='Write to the database. The default reports only.')
@with_appcontext
def import_guides(paths, userid, replace, apply):
"""Import .guide.json bundle(s) written by `guides export`."""
import os
files = []
for path in paths:
if os.path.isdir(path):
files.extend(sorted(os.path.join(path, name)
for name in os.listdir(path)
if name.endswith('.guide.json')))
else:
files.append(path)
if not files:
click.echo('No .guide.json files found.')
return
for path in files:
click.echo(os.path.basename(path))
try:
with open(path, encoding='utf-8') as handle:
payload = bundleservice.loads(handle.read())
except bundleservice.BundleError as error:
click.echo(' SKIPPED {}'.format(error))
continue
existing = Guide.query.filter(Guide.slug == payload['slug']).first()
click.echo(' title {}'.format(payload.get('title')))
click.echo(' blocks {}, images {}'.format(
len(payload['blocks']), len(payload['images'])))
if existing is not None:
click.echo(' {} already has this slug{}'.format(
'guide {}'.format(existing.guideid),
' - will be updated in place' if replace
else ' - a SECOND guide will be created; pass --replace to '
'update it instead'))
rejected = [image for image in payload['images']
if image['mimetype'] not in ALLOWEDIMAGETYPES
or len(image['data']) > MAXIMAGEBYTES]
for image in rejected:
click.echo(' DROPPED {} ({}) - not an accepted image'
.format(image['token'], image['mimetype']))
if not apply:
click.echo(' dry run, nothing written. Pass --apply to import.')
continue
try:
guide = _importbundle(payload, userid, replace, rejected)
except BlockError as error:
db.session.rollback()
click.echo(' FAILED {}'.format(error))
continue
db.session.commit()
click.echo(' imported as /guides/{} (draft, revision {})'
.format(guide.slug, guide.revision))
return [guides_cli]
@@ -156,3 +274,59 @@ def _import(parsed, name, tags, userid, replace, rejected):
tags=tags, blocks=blocks,
note='Imported from HTML', userid=userid)
return guide
def _importbundle(payload, userid, replace, rejected):
"""Create or update a guide from a bundle. Caller commits."""
guide = None
if replace:
guide = Guide.query.filter(Guide.slug == payload['slug']).first()
if guide is None:
# The bundle's own slug when it is free, because the slug is the name
# somebody pastes into a work order and the same guide should have the
# same address on every site. A clash only suffixes.
slug = payload['slug']
if Guide.query.filter(Guide.slug == slug).first() is not None:
slug = _slugify(payload.get('title') or slug)
guide = Guide(title=payload.get('title') or slug, slug=slug,
status='draft', createdby=userid)
db.session.add(guide)
db.session.flush()
# Replacing means the incoming bundle is the whole truth for this guide.
# Its blocks will reference the images arriving now, so the ones already
# stored become orphans - and re-shipping a guide monthly would otherwise
# grow a pile of them nothing points at.
if replace:
GuideImage.query.filter_by(guideid=guide.guideid).delete()
db.session.flush()
droppedtokens = {image['token'] for image in rejected}
ids = {}
for image in payload['images']:
if image['token'] in droppedtokens:
continue
row = GuideImage(guideid=guide.guideid, filename=image['filename'],
mimetype=image['mimetype'],
bytesize=len(image['data']), data=image['data'],
uploadedby=userid)
db.session.add(row)
db.session.flush()
ids[image['token']] = row.guideimageid
blocks = []
for block in payload['blocks']:
block = dict(block)
token = block.pop('imagetoken', None)
if token is not None:
if token not in ids:
continue
block['imageid'] = ids[token]
blocks.append(block)
authoring.save(guide, title=payload.get('title'),
summary=payload.get('summary'), tags=payload.get('tags'),
blocks=blocks, note='Imported from a guide bundle',
userid=userid)
return guide

130
plugins/guides/services/bundle.py Normal file
View File

@@ -0,0 +1,130 @@
"""A guide as one portable file.
WHY
Sites are separate installations with separate databases, and the useful
ones are air-gapped from where a guide gets written. A procedure that can
only exist on the machine it was typed on is the problem this plugin was
built to end, one layer up: instead of a document stranded on a share, a
guide stranded in one site's database.
So: export to a single JSON file, carry it however files get carried
there - a share, a robocopy, a USB drive - and import it on the other side.
WHAT TRAVELS
Title, summary, tags and the blocks, plus every image the blocks reference,
inline as base64. One file, no companion directory: half a guide arriving
is worse than none, and a bundle that needs its images to be copied
alongside it is a bundle that will eventually arrive without them.
WHAT DOES NOT TRAVEL
Ids, revision history, authorship and publication state. A guide's id and
its revision numbering belong to the database that issued them, and an
imported guide starts as a draft on the far side, at revision 1, owned by
whoever ran the import. The SLUG travels, because the slug is the name
somebody pastes into a work order and the whole point is that the same
guide is the same guide on both sites.
"""
import base64
import json
FORMAT = 'shopdb-guide'
VERSION = 1
# An image block points at a stored image by its id, and an id means nothing on
# the far side. Blocks travel referring to a token instead, and the importer
# swaps the tokens for the ids its own database issued.
IMAGETOKEN = 'imagetoken'
IMAGEID = 'imageid'
class BundleError(ValueError):
"""The file is not a guide bundle, or is one this version cannot read."""
def build(guide, images):
"""One guide plus its images as a plain dict, ready to be written as JSON.
`images` is the GuideImage rows for this guide. Only the ones its blocks
actually reference are carried - an orphaned paste from a draft is not part
of the procedure and does not need to cross the gap.
"""
byid = {image.guideimageid: image for image in images}
used = []
tokens = {}
blocks = []
for block in guide.blocks:
block = dict(block)
imageid = block.pop(IMAGEID, None)
if imageid is not None:
image = byid.get(imageid)
if image is None:
# The image is gone. Carrying a block that points at nothing
# would render as "Image unavailable" on the far side, which
# reads as an import failure rather than as missing data.
continue
if imageid not in tokens:
tokens[imageid] = 'img{:03d}'.format(len(used) + 1)
used.append(image)
block[IMAGETOKEN] = tokens[imageid]
blocks.append(block)
return {
'format': FORMAT,
'version': VERSION,
'slug': guide.slug,
'title': guide.title,
'summary': guide.summary,
'tags': guide.tags,
'blocks': blocks,
'images': [{
'token': tokens[image.guideimageid],
'filename': image.filename,
'mimetype': image.mimetype,
'data': base64.b64encode(image.data).decode('ascii'),
} for image in used],
}
def dumps(bundle):
return json.dumps(bundle, indent=2, sort_keys=False) + '\n'
def loads(text):
"""Parse a bundle, with the image data decoded back to bytes.
Refuses anything that is not this format. A JSON file that happens to be
somewhere near the others must not be half-imported before the failure
shows up.
"""
try:
bundle = json.loads(text)
except ValueError as error:
raise BundleError('not valid JSON: {}'.format(error))
if not isinstance(bundle, dict) or bundle.get('format') != FORMAT:
raise BundleError('not a {} file'.format(FORMAT))
version = bundle.get('version')
if not isinstance(version, int) or version > VERSION:
raise BundleError(
'bundle version {} is newer than this site can read (max {})'
.format(version, VERSION))
if not isinstance(bundle.get('blocks'), list):
raise BundleError('bundle has no blocks')
images = []
for image in bundle.get('images') or []:
try:
data = base64.b64decode(image['data'], validate=True)
except Exception:
raise BundleError('image {!r} is not readable'
.format(image.get('token')))
images.append({'token': image.get('token'),
'filename': image.get('filename'),
'mimetype': (image.get('mimetype') or '').lower(),
'data': data})
bundle['images'] = images
return bundle

146
tests/test_plugins/test_guides_bundle.py Normal file
View File

@@ -0,0 +1,146 @@
"""Carrying a guide from one site to another.
Sites are separate databases, and the ones that matter are air-gapped from
wherever the guide was written. These tests are about what has to survive that
trip and what must not: the blocks and their images travel, the ids do not.
"""
import base64
import json
import pytest
from plugins.guides.services import bundle as bundleservice
PNG = base64.b64decode(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk'
'YPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')
class FakeImage:
def __init__(self, imageid, data=PNG, mimetype='image/png',
filename='shot.png'):
self.guideimageid = imageid
self.data = data
self.mimetype = mimetype
self.filename = filename
class FakeGuide:
def __init__(self, blocks, slug='a-guide', title='A Guide',
summary='What it does', tags='shopfloor, printers'):
self.slug = slug
self.title = title
self.summary = summary
self.tags = tags
self.blocks = blocks
def test_a_bundle_carries_what_the_far_side_needs():
guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hello'}])
payload = bundleservice.build(guide, [])
assert payload['format'] == 'shopdb-guide'
assert payload['slug'] == 'a-guide'
assert payload['title'] == 'A Guide'
assert payload['summary'] == 'What it does'
assert payload['tags'] == 'shopfloor, printers'
assert payload['blocks'][0]['text'] == 'Hello'
def test_ids_do_not_travel():
"""A guide's id and revision belong to the database that issued them."""
guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hello'}])
payload = bundleservice.build(guide, [])
assert 'guideid' not in payload
assert 'revision' not in payload
assert 'publishedat' not in payload
def test_an_image_travels_inside_the_file_not_beside_it():
"""One file. A bundle that needs its images copied alongside it is a
bundle that will eventually arrive without them."""
guide = FakeGuide([{'id': 'b001', 'type': 'image', 'imageid': 42,
'alt': 'the screen'}])
payload = bundleservice.build(guide, [FakeImage(42)])
assert len(payload['images']) == 1
assert payload['images'][0]['mimetype'] == 'image/png'
assert base64.b64decode(payload['images'][0]['data']) == PNG
block = payload['blocks'][0]
assert 'imageid' not in block, 'a local id means nothing on the far side'
assert block['imagetoken'] == payload['images'][0]['token']
assert block['alt'] == 'the screen'
def test_the_same_image_used_twice_travels_once():
guide = FakeGuide([
{'id': 'b001', 'type': 'image', 'imageid': 42},
{'id': 'b002', 'type': 'image', 'imageid': 42},
])
payload = bundleservice.build(guide, [FakeImage(42)])
assert len(payload['images']) == 1
assert payload['blocks'][0]['imagetoken'] == payload['blocks'][1]['imagetoken']
def test_an_image_the_guide_does_not_use_is_left_behind():
"""An orphaned paste from a draft is not part of the procedure."""
guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hi'}])
payload = bundleservice.build(guide, [FakeImage(42), FakeImage(43)])
assert payload['images'] == []
def test_a_block_pointing_at_a_missing_image_is_dropped():
"""It would render as 'Image unavailable' on the far side, which reads as
a failed import rather than as missing data."""
guide = FakeGuide([
{'id': 'b001', 'type': 'paragraph', 'text': 'Before'},
{'id': 'b002', 'type': 'image', 'imageid': 99},
{'id': 'b003', 'type': 'paragraph', 'text': 'After'},
])
payload = bundleservice.build(guide, [])
assert [b['type'] for b in payload['blocks']] == ['paragraph', 'paragraph']
def test_a_bundle_survives_the_round_trip():
guide = FakeGuide([
{'id': 'b001', 'type': 'heading', 'text': 'Steps'},
{'id': 'b002', 'type': 'image', 'imageid': 7},
{'id': 'b003', 'type': 'checklist',
'items': [{'id': 'i1', 'text': 'Tick me'}]},
])
text = bundleservice.dumps(bundleservice.build(guide, [FakeImage(7)]))
back = bundleservice.loads(text)
assert back['slug'] == 'a-guide'
assert [b['type'] for b in back['blocks']] == ['heading', 'image', 'checklist']
assert back['images'][0]['data'] == PNG, 'decoded back to bytes, not base64'
assert back['blocks'][2]['items'][0]['text'] == 'Tick me'
def test_something_that_is_not_a_bundle_is_refused_before_anything_is_written():
with pytest.raises(bundleservice.BundleError):
bundleservice.loads('{"hello": "world"}')
with pytest.raises(bundleservice.BundleError):
bundleservice.loads('not json at all')
with pytest.raises(bundleservice.BundleError):
bundleservice.loads(json.dumps({'format': 'shopdb-guide', 'version': 1}))
def test_a_bundle_from_a_newer_site_is_refused_rather_than_half_read():
newer = json.dumps({'format': 'shopdb-guide', 'version': 99,
'slug': 'x', 'blocks': []})
with pytest.raises(bundleservice.BundleError) as error:
bundleservice.loads(newer)
assert 'newer than this site can read' in str(error.value)
def test_an_unreadable_image_names_itself():
broken = json.dumps({
'format': 'shopdb-guide', 'version': 1, 'slug': 'x', 'blocks': [],
'images': [{'token': 'img001', 'mimetype': 'image/png',
'filename': 'x.png', 'data': 'not base64 !!'}]})
with pytest.raises(bundleservice.BundleError) as error:
bundleservice.loads(broken)
assert 'img001' in str(error.value)