diff --git a/CHANGELOG.md b/CHANGELOG.md index ecf43009..1a9425fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,19 @@ ADR-007 and ADR-002. tables; in self-hosted mode one stored loan expands into a take and a return, so a returned drive is distinguishable from one still out. +- **`flask guides export` / `flask guides import` - a guide can reach another + site.** Sites are separate databases and the ones that matter are air-gapped + from wherever a guide gets written, so a procedure written once was stranded + where it was typed. Export writes one `.guide.json` per guide with its images + carried inside it - one file, because a bundle that needs its images copied + alongside it is a bundle that will eventually arrive without them - and + import reads them back. The slug travels, since that is the address somebody + pastes into a work order and the same guide should answer to the same name on + every site; ids, revision history and publication state do not, because those + belong to the database that issued them. Imports arrive as drafts; + `--replace` updates a guide already carrying that slug rather than creating a + second one, which is how a corrected guide is re-shipped. + - **`flask guides import-html` - an existing HTML article becomes a guide.** A site adopting this app arrives holding procedures already written: styled HTML pages that get mailed around and opened off a share. Retyping one into diff --git a/plugins/guides/cli.py b/plugins/guides/cli.py index 0fde0a72..6b02b89b 100644 --- a/plugins/guides/cli.py +++ b/plugins/guides/cli.py @@ -1,15 +1,22 @@ -"""CLI: bring an existing HTML article in as a guide. +"""CLI: move guides in and out of a site. - flask guides import-html PATH [PATH ...] + flask guides import-html PATH [PATH ...] an HTML article becomes a guide + flask guides export --out DIR guides become portable files + flask guides import PATH [PATH ...] those files become guides here A site that adopts this app usually arrives with a folder of procedures already written - styled HTML pages that get mailed around and opened off a share. This imports them so they live where they are used, become searchable, tickable and printable, and stop being a file somebody has to find. -Imports land as DRAFTS, always. The conversion is faithful but it is not a -render: layout, colour and anything decorative are dropped on purpose, and a -person should read the result before anybody is told to follow it. +Imports land as DRAFTS, always - both kinds. An HTML conversion is faithful but +it is not a render, and a bundle written on another site has not been read by +anybody here. Publishing is a decision somebody makes after reading. + +Sites are separate databases and the useful ones are air-gapped from where a +guide gets written, so export/import is how the same procedure reaches all of +them: one JSON file per guide, images carried inside it, moved by whatever +moves files to that site. """ import click @@ -18,7 +25,7 @@ from flask.cli import AppGroup, with_appcontext from shopdb.api import db from .models import Guide, GuideImage -from .services import authoring +from .services import authoring, bundle as bundleservice from .services.blocks import BlockError from .services.htmlimport import parse @@ -107,6 +114,117 @@ def build_cli(): click.echo(' imported as /guides/{} (draft, revision {})' .format(guide.slug, guide.revision)) + @guides_cli.command('export') + @click.option('--out', 'outdir', required=True, + type=click.Path(file_okay=False), + help='Directory to write the bundles into. Created if needed.') + @click.option('--slug', 'slugs', multiple=True, + help='Export just this guide. Repeatable. Default: all.') + @click.option('--published-only', is_flag=True, + help='Skip guides that have never been published.') + @with_appcontext + def export_guides(outdir, slugs, published_only): + """Write guides out as portable .guide.json files.""" + import os + + query = Guide.query + if slugs: + query = query.filter(Guide.slug.in_(slugs)) + if published_only: + query = query.filter(Guide.publishedrevisionid.isnot(None)) + guides = query.order_by(Guide.slug).all() + + missing = set(slugs) - {guide.slug for guide in guides} + for slug in sorted(missing): + click.echo(' NOT FOUND {}'.format(slug)) + + if not guides: + click.echo('Nothing to export.') + return + + os.makedirs(outdir, exist_ok=True) + for guide in guides: + images = GuideImage.query.filter_by(guideid=guide.guideid).all() + payload = bundleservice.build(guide, images) + path = os.path.join(outdir, '{}.guide.json'.format(guide.slug)) + with open(path, 'w', encoding='utf-8') as handle: + handle.write(bundleservice.dumps(payload)) + click.echo(' {} {} blocks, {} image(s), {:.0f} KB{}'.format( + os.path.basename(path), len(payload['blocks']), + len(payload['images']), os.path.getsize(path) / 1024, + '' if guide.publishedrevisionid else ' [draft here]')) + click.echo('{} guide(s) -> {}'.format(len(guides), outdir)) + + @guides_cli.command('import') + @click.argument('paths', nargs=-1, required=True, + type=click.Path(exists=True)) + @click.option('--userid', type=int, default=None, + help='Record this user as the author on this site.') + @click.option('--replace', is_flag=True, + help='If a guide already has the bundle\'s slug, edit it in ' + 'place (a new revision) instead of creating a second ' + 'one. This is how an updated guide is re-shipped.') + @click.option('--apply/--dry-run', default=False, + help='Write to the database. The default reports only.') + @with_appcontext + def import_guides(paths, userid, replace, apply): + """Import .guide.json bundle(s) written by `guides export`.""" + import os + + files = [] + for path in paths: + if os.path.isdir(path): + files.extend(sorted(os.path.join(path, name) + for name in os.listdir(path) + if name.endswith('.guide.json'))) + else: + files.append(path) + if not files: + click.echo('No .guide.json files found.') + return + + for path in files: + click.echo(os.path.basename(path)) + try: + with open(path, encoding='utf-8') as handle: + payload = bundleservice.loads(handle.read()) + except bundleservice.BundleError as error: + click.echo(' SKIPPED {}'.format(error)) + continue + + existing = Guide.query.filter(Guide.slug == payload['slug']).first() + click.echo(' title {}'.format(payload.get('title'))) + click.echo(' blocks {}, images {}'.format( + len(payload['blocks']), len(payload['images']))) + if existing is not None: + click.echo(' {} already has this slug{}'.format( + 'guide {}'.format(existing.guideid), + ' - will be updated in place' if replace + else ' - a SECOND guide will be created; pass --replace to ' + 'update it instead')) + + rejected = [image for image in payload['images'] + if image['mimetype'] not in ALLOWEDIMAGETYPES + or len(image['data']) > MAXIMAGEBYTES] + for image in rejected: + click.echo(' DROPPED {} ({}) - not an accepted image' + .format(image['token'], image['mimetype'])) + + if not apply: + click.echo(' dry run, nothing written. Pass --apply to import.') + continue + + try: + guide = _importbundle(payload, userid, replace, rejected) + except BlockError as error: + db.session.rollback() + click.echo(' FAILED {}'.format(error)) + continue + + db.session.commit() + click.echo(' imported as /guides/{} (draft, revision {})' + .format(guide.slug, guide.revision)) + return [guides_cli] @@ -156,3 +274,59 @@ def _import(parsed, name, tags, userid, replace, rejected): tags=tags, blocks=blocks, note='Imported from HTML', userid=userid) return guide + + +def _importbundle(payload, userid, replace, rejected): + """Create or update a guide from a bundle. Caller commits.""" + guide = None + if replace: + guide = Guide.query.filter(Guide.slug == payload['slug']).first() + + if guide is None: + # The bundle's own slug when it is free, because the slug is the name + # somebody pastes into a work order and the same guide should have the + # same address on every site. A clash only suffixes. + slug = payload['slug'] + if Guide.query.filter(Guide.slug == slug).first() is not None: + slug = _slugify(payload.get('title') or slug) + guide = Guide(title=payload.get('title') or slug, slug=slug, + status='draft', createdby=userid) + db.session.add(guide) + db.session.flush() + + # Replacing means the incoming bundle is the whole truth for this guide. + # Its blocks will reference the images arriving now, so the ones already + # stored become orphans - and re-shipping a guide monthly would otherwise + # grow a pile of them nothing points at. + if replace: + GuideImage.query.filter_by(guideid=guide.guideid).delete() + db.session.flush() + + droppedtokens = {image['token'] for image in rejected} + ids = {} + for image in payload['images']: + if image['token'] in droppedtokens: + continue + row = GuideImage(guideid=guide.guideid, filename=image['filename'], + mimetype=image['mimetype'], + bytesize=len(image['data']), data=image['data'], + uploadedby=userid) + db.session.add(row) + db.session.flush() + ids[image['token']] = row.guideimageid + + blocks = [] + for block in payload['blocks']: + block = dict(block) + token = block.pop('imagetoken', None) + if token is not None: + if token not in ids: + continue + block['imageid'] = ids[token] + blocks.append(block) + + authoring.save(guide, title=payload.get('title'), + summary=payload.get('summary'), tags=payload.get('tags'), + blocks=blocks, note='Imported from a guide bundle', + userid=userid) + return guide diff --git a/plugins/guides/services/bundle.py b/plugins/guides/services/bundle.py new file mode 100644 index 00000000..488bcade --- /dev/null +++ b/plugins/guides/services/bundle.py @@ -0,0 +1,130 @@ +"""A guide as one portable file. + +WHY + + Sites are separate installations with separate databases, and the useful + ones are air-gapped from where a guide gets written. A procedure that can + only exist on the machine it was typed on is the problem this plugin was + built to end, one layer up: instead of a document stranded on a share, a + guide stranded in one site's database. + + So: export to a single JSON file, carry it however files get carried + there - a share, a robocopy, a USB drive - and import it on the other side. + +WHAT TRAVELS + + Title, summary, tags and the blocks, plus every image the blocks reference, + inline as base64. One file, no companion directory: half a guide arriving + is worse than none, and a bundle that needs its images to be copied + alongside it is a bundle that will eventually arrive without them. + +WHAT DOES NOT TRAVEL + + Ids, revision history, authorship and publication state. A guide's id and + its revision numbering belong to the database that issued them, and an + imported guide starts as a draft on the far side, at revision 1, owned by + whoever ran the import. The SLUG travels, because the slug is the name + somebody pastes into a work order and the whole point is that the same + guide is the same guide on both sites. +""" + +import base64 +import json + +FORMAT = 'shopdb-guide' +VERSION = 1 + +# An image block points at a stored image by its id, and an id means nothing on +# the far side. Blocks travel referring to a token instead, and the importer +# swaps the tokens for the ids its own database issued. +IMAGETOKEN = 'imagetoken' +IMAGEID = 'imageid' + + +class BundleError(ValueError): + """The file is not a guide bundle, or is one this version cannot read.""" + + +def build(guide, images): + """One guide plus its images as a plain dict, ready to be written as JSON. + + `images` is the GuideImage rows for this guide. Only the ones its blocks + actually reference are carried - an orphaned paste from a draft is not part + of the procedure and does not need to cross the gap. + """ + byid = {image.guideimageid: image for image in images} + used = [] + tokens = {} + blocks = [] + for block in guide.blocks: + block = dict(block) + imageid = block.pop(IMAGEID, None) + if imageid is not None: + image = byid.get(imageid) + if image is None: + # The image is gone. Carrying a block that points at nothing + # would render as "Image unavailable" on the far side, which + # reads as an import failure rather than as missing data. + continue + if imageid not in tokens: + tokens[imageid] = 'img{:03d}'.format(len(used) + 1) + used.append(image) + block[IMAGETOKEN] = tokens[imageid] + blocks.append(block) + + return { + 'format': FORMAT, + 'version': VERSION, + 'slug': guide.slug, + 'title': guide.title, + 'summary': guide.summary, + 'tags': guide.tags, + 'blocks': blocks, + 'images': [{ + 'token': tokens[image.guideimageid], + 'filename': image.filename, + 'mimetype': image.mimetype, + 'data': base64.b64encode(image.data).decode('ascii'), + } for image in used], + } + + +def dumps(bundle): + return json.dumps(bundle, indent=2, sort_keys=False) + '\n' + + +def loads(text): + """Parse a bundle, with the image data decoded back to bytes. + + Refuses anything that is not this format. A JSON file that happens to be + somewhere near the others must not be half-imported before the failure + shows up. + """ + try: + bundle = json.loads(text) + except ValueError as error: + raise BundleError('not valid JSON: {}'.format(error)) + + if not isinstance(bundle, dict) or bundle.get('format') != FORMAT: + raise BundleError('not a {} file'.format(FORMAT)) + version = bundle.get('version') + if not isinstance(version, int) or version > VERSION: + raise BundleError( + 'bundle version {} is newer than this site can read (max {})' + .format(version, VERSION)) + if not isinstance(bundle.get('blocks'), list): + raise BundleError('bundle has no blocks') + + images = [] + for image in bundle.get('images') or []: + try: + data = base64.b64decode(image['data'], validate=True) + except Exception: + raise BundleError('image {!r} is not readable' + .format(image.get('token'))) + images.append({'token': image.get('token'), + 'filename': image.get('filename'), + 'mimetype': (image.get('mimetype') or '').lower(), + 'data': data}) + bundle['images'] = images + return bundle diff --git a/tests/test_plugins/test_guides_bundle.py b/tests/test_plugins/test_guides_bundle.py new file mode 100644 index 00000000..231b42f7 --- /dev/null +++ b/tests/test_plugins/test_guides_bundle.py @@ -0,0 +1,146 @@ +"""Carrying a guide from one site to another. + +Sites are separate databases, and the ones that matter are air-gapped from +wherever the guide was written. These tests are about what has to survive that +trip and what must not: the blocks and their images travel, the ids do not. +""" + +import base64 +import json + +import pytest + +from plugins.guides.services import bundle as bundleservice + + +PNG = base64.b64decode( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk' + 'YPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==') + + +class FakeImage: + def __init__(self, imageid, data=PNG, mimetype='image/png', + filename='shot.png'): + self.guideimageid = imageid + self.data = data + self.mimetype = mimetype + self.filename = filename + + +class FakeGuide: + def __init__(self, blocks, slug='a-guide', title='A Guide', + summary='What it does', tags='shopfloor, printers'): + self.slug = slug + self.title = title + self.summary = summary + self.tags = tags + self.blocks = blocks + + +def test_a_bundle_carries_what_the_far_side_needs(): + guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hello'}]) + payload = bundleservice.build(guide, []) + assert payload['format'] == 'shopdb-guide' + assert payload['slug'] == 'a-guide' + assert payload['title'] == 'A Guide' + assert payload['summary'] == 'What it does' + assert payload['tags'] == 'shopfloor, printers' + assert payload['blocks'][0]['text'] == 'Hello' + + +def test_ids_do_not_travel(): + """A guide's id and revision belong to the database that issued them.""" + guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hello'}]) + payload = bundleservice.build(guide, []) + assert 'guideid' not in payload + assert 'revision' not in payload + assert 'publishedat' not in payload + + +def test_an_image_travels_inside_the_file_not_beside_it(): + """One file. A bundle that needs its images copied alongside it is a + bundle that will eventually arrive without them.""" + guide = FakeGuide([{'id': 'b001', 'type': 'image', 'imageid': 42, + 'alt': 'the screen'}]) + payload = bundleservice.build(guide, [FakeImage(42)]) + + assert len(payload['images']) == 1 + assert payload['images'][0]['mimetype'] == 'image/png' + assert base64.b64decode(payload['images'][0]['data']) == PNG + + block = payload['blocks'][0] + assert 'imageid' not in block, 'a local id means nothing on the far side' + assert block['imagetoken'] == payload['images'][0]['token'] + assert block['alt'] == 'the screen' + + +def test_the_same_image_used_twice_travels_once(): + guide = FakeGuide([ + {'id': 'b001', 'type': 'image', 'imageid': 42}, + {'id': 'b002', 'type': 'image', 'imageid': 42}, + ]) + payload = bundleservice.build(guide, [FakeImage(42)]) + assert len(payload['images']) == 1 + assert payload['blocks'][0]['imagetoken'] == payload['blocks'][1]['imagetoken'] + + +def test_an_image_the_guide_does_not_use_is_left_behind(): + """An orphaned paste from a draft is not part of the procedure.""" + guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hi'}]) + payload = bundleservice.build(guide, [FakeImage(42), FakeImage(43)]) + assert payload['images'] == [] + + +def test_a_block_pointing_at_a_missing_image_is_dropped(): + """It would render as 'Image unavailable' on the far side, which reads as + a failed import rather than as missing data.""" + guide = FakeGuide([ + {'id': 'b001', 'type': 'paragraph', 'text': 'Before'}, + {'id': 'b002', 'type': 'image', 'imageid': 99}, + {'id': 'b003', 'type': 'paragraph', 'text': 'After'}, + ]) + payload = bundleservice.build(guide, []) + assert [b['type'] for b in payload['blocks']] == ['paragraph', 'paragraph'] + + +def test_a_bundle_survives_the_round_trip(): + guide = FakeGuide([ + {'id': 'b001', 'type': 'heading', 'text': 'Steps'}, + {'id': 'b002', 'type': 'image', 'imageid': 7}, + {'id': 'b003', 'type': 'checklist', + 'items': [{'id': 'i1', 'text': 'Tick me'}]}, + ]) + text = bundleservice.dumps(bundleservice.build(guide, [FakeImage(7)])) + back = bundleservice.loads(text) + + assert back['slug'] == 'a-guide' + assert [b['type'] for b in back['blocks']] == ['heading', 'image', 'checklist'] + assert back['images'][0]['data'] == PNG, 'decoded back to bytes, not base64' + assert back['blocks'][2]['items'][0]['text'] == 'Tick me' + + +def test_something_that_is_not_a_bundle_is_refused_before_anything_is_written(): + with pytest.raises(bundleservice.BundleError): + bundleservice.loads('{"hello": "world"}') + with pytest.raises(bundleservice.BundleError): + bundleservice.loads('not json at all') + with pytest.raises(bundleservice.BundleError): + bundleservice.loads(json.dumps({'format': 'shopdb-guide', 'version': 1})) + + +def test_a_bundle_from_a_newer_site_is_refused_rather_than_half_read(): + newer = json.dumps({'format': 'shopdb-guide', 'version': 99, + 'slug': 'x', 'blocks': []}) + with pytest.raises(bundleservice.BundleError) as error: + bundleservice.loads(newer) + assert 'newer than this site can read' in str(error.value) + + +def test_an_unreadable_image_names_itself(): + broken = json.dumps({ + 'format': 'shopdb-guide', 'version': 1, 'slug': 'x', 'blocks': [], + 'images': [{'token': 'img001', 'mimetype': 'image/png', + 'filename': 'x.png', 'data': 'not base64 !!'}]}) + with pytest.raises(bundleservice.BundleError) as error: + bundleservice.loads(broken) + assert 'img001' in str(error.value)