guides export/import bundles

CHANGELOG.md

@@ -22,6 +22,19 @@ ADR-007 and ADR-002.
   tables; in self-hosted mode one stored loan expands into a take and a return,
   so a returned drive is distinguishable from one still out.
 
+- **`flask guides export` / `flask guides import` - a guide can reach another
+  site.** Sites are separate databases and the ones that matter are air-gapped
+  from wherever a guide gets written, so a procedure written once was stranded
+  where it was typed. Export writes one `.guide.json` per guide with its images
+  carried inside it - one file, because a bundle that needs its images copied
+  alongside it is a bundle that will eventually arrive without them - and
+  import reads them back. The slug travels, since that is the address somebody
+  pastes into a work order and the same guide should answer to the same name on
+  every site; ids, revision history and publication state do not, because those
+  belong to the database that issued them. Imports arrive as drafts;
+  `--replace` updates a guide already carrying that slug rather than creating a
+  second one, which is how a corrected guide is re-shipped.
+
 - **`flask guides import-html` - an existing HTML article becomes a guide.**
   A site adopting this app arrives holding procedures already written: styled
   HTML pages that get mailed around and opened off a share. Retyping one into

plugins/guides/cli.py

@@ -1,15 +1,22 @@
-"""CLI: bring an existing HTML article in as a guide.
+"""CLI: move guides in and out of a site.
 
-    flask guides import-html PATH [PATH ...]
+    flask guides import-html PATH [PATH ...]   an HTML article becomes a guide
+    flask guides export --out DIR              guides become portable files
+    flask guides import PATH [PATH ...]        those files become guides here
 
 A site that adopts this app usually arrives with a folder of procedures already
 written - styled HTML pages that get mailed around and opened off a share. This
 imports them so they live where they are used, become searchable, tickable and
 printable, and stop being a file somebody has to find.
 
-Imports land as DRAFTS, always. The conversion is faithful but it is not a
+Imports land as DRAFTS, always - both kinds. An HTML conversion is faithful but
-render: layout, colour and anything decorative are dropped on purpose, and a
+it is not a render, and a bundle written on another site has not been read by
-person should read the result before anybody is told to follow it.
+anybody here. Publishing is a decision somebody makes after reading.
+
+Sites are separate databases and the useful ones are air-gapped from where a
+guide gets written, so export/import is how the same procedure reaches all of
+them: one JSON file per guide, images carried inside it, moved by whatever
+moves files to that site.
 """
 
 import click
@@ -18,7 +25,7 @@ from flask.cli import AppGroup, with_appcontext
 from shopdb.api import db
 
 from .models import Guide, GuideImage
-from .services import authoring
+from .services import authoring, bundle as bundleservice
 from .services.blocks import BlockError
 from .services.htmlimport import parse
 
@@ -107,6 +114,117 @@ def build_cli():
             click.echo('  imported as /guides/{} (draft, revision {})'
                        .format(guide.slug, guide.revision))
 
+    @guides_cli.command('export')
+    @click.option('--out', 'outdir', required=True,
+                  type=click.Path(file_okay=False),
+                  help='Directory to write the bundles into. Created if needed.')
+    @click.option('--slug', 'slugs', multiple=True,
+                  help='Export just this guide. Repeatable. Default: all.')
+    @click.option('--published-only', is_flag=True,
+                  help='Skip guides that have never been published.')
+    @with_appcontext
+    def export_guides(outdir, slugs, published_only):
+        """Write guides out as portable .guide.json files."""
+        import os
+
+        query = Guide.query
+        if slugs:
+            query = query.filter(Guide.slug.in_(slugs))
+        if published_only:
+            query = query.filter(Guide.publishedrevisionid.isnot(None))
+        guides = query.order_by(Guide.slug).all()
+
+        missing = set(slugs) - {guide.slug for guide in guides}
+        for slug in sorted(missing):
+            click.echo('  NOT FOUND {}'.format(slug))
+
+        if not guides:
+            click.echo('Nothing to export.')
+            return
+
+        os.makedirs(outdir, exist_ok=True)
+        for guide in guides:
+            images = GuideImage.query.filter_by(guideid=guide.guideid).all()
+            payload = bundleservice.build(guide, images)
+            path = os.path.join(outdir, '{}.guide.json'.format(guide.slug))
+            with open(path, 'w', encoding='utf-8') as handle:
+                handle.write(bundleservice.dumps(payload))
+            click.echo('  {}  {} blocks, {} image(s), {:.0f} KB{}'.format(
+                os.path.basename(path), len(payload['blocks']),
+                len(payload['images']), os.path.getsize(path) / 1024,
+                '' if guide.publishedrevisionid else '  [draft here]'))
+        click.echo('{} guide(s) -> {}'.format(len(guides), outdir))
+
+    @guides_cli.command('import')
+    @click.argument('paths', nargs=-1, required=True,
+                    type=click.Path(exists=True))
+    @click.option('--userid', type=int, default=None,
+                  help='Record this user as the author on this site.')
+    @click.option('--replace', is_flag=True,
+                  help='If a guide already has the bundle\'s slug, edit it in '
+                       'place (a new revision) instead of creating a second '
+                       'one. This is how an updated guide is re-shipped.')
+    @click.option('--apply/--dry-run', default=False,
+                  help='Write to the database. The default reports only.')
+    @with_appcontext
+    def import_guides(paths, userid, replace, apply):
+        """Import .guide.json bundle(s) written by `guides export`."""
+        import os
+
+        files = []
+        for path in paths:
+            if os.path.isdir(path):
+                files.extend(sorted(os.path.join(path, name)
+                                    for name in os.listdir(path)
+                                    if name.endswith('.guide.json')))
+            else:
+                files.append(path)
+        if not files:
+            click.echo('No .guide.json files found.')
+            return
+
+        for path in files:
+            click.echo(os.path.basename(path))
+            try:
+                with open(path, encoding='utf-8') as handle:
+                    payload = bundleservice.loads(handle.read())
+            except bundleservice.BundleError as error:
+                click.echo('  SKIPPED {}'.format(error))
+                continue
+
+            existing = Guide.query.filter(Guide.slug == payload['slug']).first()
+            click.echo('  title   {}'.format(payload.get('title')))
+            click.echo('  blocks  {}, images {}'.format(
+                len(payload['blocks']), len(payload['images'])))
+            if existing is not None:
+                click.echo('  {} already has this slug{}'.format(
+                    'guide {}'.format(existing.guideid),
+                    ' - will be updated in place' if replace
+                    else ' - a SECOND guide will be created; pass --replace to '
+                         'update it instead'))
+
+            rejected = [image for image in payload['images']
+                        if image['mimetype'] not in ALLOWEDIMAGETYPES
+                        or len(image['data']) > MAXIMAGEBYTES]
+            for image in rejected:
+                click.echo('  DROPPED {} ({}) - not an accepted image'
+                           .format(image['token'], image['mimetype']))
+
+            if not apply:
+                click.echo('  dry run, nothing written. Pass --apply to import.')
+                continue
+
+            try:
+                guide = _importbundle(payload, userid, replace, rejected)
+            except BlockError as error:
+                db.session.rollback()
+                click.echo('  FAILED  {}'.format(error))
+                continue
+
+            db.session.commit()
+            click.echo('  imported as /guides/{} (draft, revision {})'
+                       .format(guide.slug, guide.revision))
+
     return [guides_cli]
 
 
@@ -156,3 +274,59 @@ def _import(parsed, name, tags, userid, replace, rejected):
                    tags=tags, blocks=blocks,
                    note='Imported from HTML', userid=userid)
     return guide
+
+
+def _importbundle(payload, userid, replace, rejected):
+    """Create or update a guide from a bundle. Caller commits."""
+    guide = None
+    if replace:
+        guide = Guide.query.filter(Guide.slug == payload['slug']).first()
+
+    if guide is None:
+        # The bundle's own slug when it is free, because the slug is the name
+        # somebody pastes into a work order and the same guide should have the
+        # same address on every site. A clash only suffixes.
+        slug = payload['slug']
+        if Guide.query.filter(Guide.slug == slug).first() is not None:
+            slug = _slugify(payload.get('title') or slug)
+        guide = Guide(title=payload.get('title') or slug, slug=slug,
+                      status='draft', createdby=userid)
+        db.session.add(guide)
+        db.session.flush()
+
+    # Replacing means the incoming bundle is the whole truth for this guide.
+    # Its blocks will reference the images arriving now, so the ones already
+    # stored become orphans - and re-shipping a guide monthly would otherwise
+    # grow a pile of them nothing points at.
+    if replace:
+        GuideImage.query.filter_by(guideid=guide.guideid).delete()
+        db.session.flush()
+
+    droppedtokens = {image['token'] for image in rejected}
+    ids = {}
+    for image in payload['images']:
+        if image['token'] in droppedtokens:
+            continue
+        row = GuideImage(guideid=guide.guideid, filename=image['filename'],
+                         mimetype=image['mimetype'],
+                         bytesize=len(image['data']), data=image['data'],
+                         uploadedby=userid)
+        db.session.add(row)
+        db.session.flush()
+        ids[image['token']] = row.guideimageid
+
+    blocks = []
+    for block in payload['blocks']:
+        block = dict(block)
+        token = block.pop('imagetoken', None)
+        if token is not None:
+            if token not in ids:
+                continue
+            block['imageid'] = ids[token]
+        blocks.append(block)
+
+    authoring.save(guide, title=payload.get('title'),
+                   summary=payload.get('summary'), tags=payload.get('tags'),
+                   blocks=blocks, note='Imported from a guide bundle',
+                   userid=userid)
+    return guide

plugins/guides/services/bundle.py

@@ -0,0 +1,130 @@
+"""A guide as one portable file.
+
+WHY
+
+    Sites are separate installations with separate databases, and the useful
+    ones are air-gapped from where a guide gets written. A procedure that can
+    only exist on the machine it was typed on is the problem this plugin was
+    built to end, one layer up: instead of a document stranded on a share, a
+    guide stranded in one site's database.
+
+    So: export to a single JSON file, carry it however files get carried
+    there - a share, a robocopy, a USB drive - and import it on the other side.
+
+WHAT TRAVELS
+
+    Title, summary, tags and the blocks, plus every image the blocks reference,
+    inline as base64. One file, no companion directory: half a guide arriving
+    is worse than none, and a bundle that needs its images to be copied
+    alongside it is a bundle that will eventually arrive without them.
+
+WHAT DOES NOT TRAVEL
+
+    Ids, revision history, authorship and publication state. A guide's id and
+    its revision numbering belong to the database that issued them, and an
+    imported guide starts as a draft on the far side, at revision 1, owned by
+    whoever ran the import. The SLUG travels, because the slug is the name
+    somebody pastes into a work order and the whole point is that the same
+    guide is the same guide on both sites.
+"""
+
+import base64
+import json
+
+FORMAT = 'shopdb-guide'
+VERSION = 1
+
+# An image block points at a stored image by its id, and an id means nothing on
+# the far side. Blocks travel referring to a token instead, and the importer
+# swaps the tokens for the ids its own database issued.
+IMAGETOKEN = 'imagetoken'
+IMAGEID = 'imageid'
+
+
+class BundleError(ValueError):
+    """The file is not a guide bundle, or is one this version cannot read."""
+
+
+def build(guide, images):
+    """One guide plus its images as a plain dict, ready to be written as JSON.
+
+    `images` is the GuideImage rows for this guide. Only the ones its blocks
+    actually reference are carried - an orphaned paste from a draft is not part
+    of the procedure and does not need to cross the gap.
+    """
+    byid = {image.guideimageid: image for image in images}
+    used = []
+    tokens = {}
+    blocks = []
+    for block in guide.blocks:
+        block = dict(block)
+        imageid = block.pop(IMAGEID, None)
+        if imageid is not None:
+            image = byid.get(imageid)
+            if image is None:
+                # The image is gone. Carrying a block that points at nothing
+                # would render as "Image unavailable" on the far side, which
+                # reads as an import failure rather than as missing data.
+                continue
+            if imageid not in tokens:
+                tokens[imageid] = 'img{:03d}'.format(len(used) + 1)
+                used.append(image)
+            block[IMAGETOKEN] = tokens[imageid]
+        blocks.append(block)
+
+    return {
+        'format': FORMAT,
+        'version': VERSION,
+        'slug': guide.slug,
+        'title': guide.title,
+        'summary': guide.summary,
+        'tags': guide.tags,
+        'blocks': blocks,
+        'images': [{
+            'token': tokens[image.guideimageid],
+            'filename': image.filename,
+            'mimetype': image.mimetype,
+            'data': base64.b64encode(image.data).decode('ascii'),
+        } for image in used],
+    }
+
+
+def dumps(bundle):
+    return json.dumps(bundle, indent=2, sort_keys=False) + '\n'
+
+
+def loads(text):
+    """Parse a bundle, with the image data decoded back to bytes.
+
+    Refuses anything that is not this format. A JSON file that happens to be
+    somewhere near the others must not be half-imported before the failure
+    shows up.
+    """
+    try:
+        bundle = json.loads(text)
+    except ValueError as error:
+        raise BundleError('not valid JSON: {}'.format(error))
+
+    if not isinstance(bundle, dict) or bundle.get('format') != FORMAT:
+        raise BundleError('not a {} file'.format(FORMAT))
+    version = bundle.get('version')
+    if not isinstance(version, int) or version > VERSION:
+        raise BundleError(
+            'bundle version {} is newer than this site can read (max {})'
+            .format(version, VERSION))
+    if not isinstance(bundle.get('blocks'), list):
+        raise BundleError('bundle has no blocks')
+
+    images = []
+    for image in bundle.get('images') or []:
+        try:
+            data = base64.b64decode(image['data'], validate=True)
+        except Exception:
+            raise BundleError('image {!r} is not readable'
+                              .format(image.get('token')))
+        images.append({'token': image.get('token'),
+                       'filename': image.get('filename'),
+                       'mimetype': (image.get('mimetype') or '').lower(),
+                       'data': data})
+    bundle['images'] = images
+    return bundle

tests/test_plugins/test_guides_bundle.py

@@ -0,0 +1,146 @@
+"""Carrying a guide from one site to another.
+
+Sites are separate databases, and the ones that matter are air-gapped from
+wherever the guide was written. These tests are about what has to survive that
+trip and what must not: the blocks and their images travel, the ids do not.
+"""
+
+import base64
+import json
+
+import pytest
+
+from plugins.guides.services import bundle as bundleservice
+
+
+PNG = base64.b64decode(
+    'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk'
+    'YPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')
+
+
+class FakeImage:
+    def __init__(self, imageid, data=PNG, mimetype='image/png',
+                 filename='shot.png'):
+        self.guideimageid = imageid
+        self.data = data
+        self.mimetype = mimetype
+        self.filename = filename
+
+
+class FakeGuide:
+    def __init__(self, blocks, slug='a-guide', title='A Guide',
+                 summary='What it does', tags='shopfloor, printers'):
+        self.slug = slug
+        self.title = title
+        self.summary = summary
+        self.tags = tags
+        self.blocks = blocks
+
+
+def test_a_bundle_carries_what_the_far_side_needs():
+    guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hello'}])
+    payload = bundleservice.build(guide, [])
+    assert payload['format'] == 'shopdb-guide'
+    assert payload['slug'] == 'a-guide'
+    assert payload['title'] == 'A Guide'
+    assert payload['summary'] == 'What it does'
+    assert payload['tags'] == 'shopfloor, printers'
+    assert payload['blocks'][0]['text'] == 'Hello'
+
+
+def test_ids_do_not_travel():
+    """A guide's id and revision belong to the database that issued them."""
+    guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hello'}])
+    payload = bundleservice.build(guide, [])
+    assert 'guideid' not in payload
+    assert 'revision' not in payload
+    assert 'publishedat' not in payload
+
+
+def test_an_image_travels_inside_the_file_not_beside_it():
+    """One file. A bundle that needs its images copied alongside it is a
+    bundle that will eventually arrive without them."""
+    guide = FakeGuide([{'id': 'b001', 'type': 'image', 'imageid': 42,
+                        'alt': 'the screen'}])
+    payload = bundleservice.build(guide, [FakeImage(42)])
+
+    assert len(payload['images']) == 1
+    assert payload['images'][0]['mimetype'] == 'image/png'
+    assert base64.b64decode(payload['images'][0]['data']) == PNG
+
+    block = payload['blocks'][0]
+    assert 'imageid' not in block, 'a local id means nothing on the far side'
+    assert block['imagetoken'] == payload['images'][0]['token']
+    assert block['alt'] == 'the screen'
+
+
+def test_the_same_image_used_twice_travels_once():
+    guide = FakeGuide([
+        {'id': 'b001', 'type': 'image', 'imageid': 42},
+        {'id': 'b002', 'type': 'image', 'imageid': 42},
+    ])
+    payload = bundleservice.build(guide, [FakeImage(42)])
+    assert len(payload['images']) == 1
+    assert payload['blocks'][0]['imagetoken'] == payload['blocks'][1]['imagetoken']
+
+
+def test_an_image_the_guide_does_not_use_is_left_behind():
+    """An orphaned paste from a draft is not part of the procedure."""
+    guide = FakeGuide([{'id': 'b001', 'type': 'paragraph', 'text': 'Hi'}])
+    payload = bundleservice.build(guide, [FakeImage(42), FakeImage(43)])
+    assert payload['images'] == []
+
+
+def test_a_block_pointing_at_a_missing_image_is_dropped():
+    """It would render as 'Image unavailable' on the far side, which reads as
+    a failed import rather than as missing data."""
+    guide = FakeGuide([
+        {'id': 'b001', 'type': 'paragraph', 'text': 'Before'},
+        {'id': 'b002', 'type': 'image', 'imageid': 99},
+        {'id': 'b003', 'type': 'paragraph', 'text': 'After'},
+    ])
+    payload = bundleservice.build(guide, [])
+    assert [b['type'] for b in payload['blocks']] == ['paragraph', 'paragraph']
+
+
+def test_a_bundle_survives_the_round_trip():
+    guide = FakeGuide([
+        {'id': 'b001', 'type': 'heading', 'text': 'Steps'},
+        {'id': 'b002', 'type': 'image', 'imageid': 7},
+        {'id': 'b003', 'type': 'checklist',
+         'items': [{'id': 'i1', 'text': 'Tick me'}]},
+    ])
+    text = bundleservice.dumps(bundleservice.build(guide, [FakeImage(7)]))
+    back = bundleservice.loads(text)
+
+    assert back['slug'] == 'a-guide'
+    assert [b['type'] for b in back['blocks']] == ['heading', 'image', 'checklist']
+    assert back['images'][0]['data'] == PNG, 'decoded back to bytes, not base64'
+    assert back['blocks'][2]['items'][0]['text'] == 'Tick me'
+
+
+def test_something_that_is_not_a_bundle_is_refused_before_anything_is_written():
+    with pytest.raises(bundleservice.BundleError):
+        bundleservice.loads('{"hello": "world"}')
+    with pytest.raises(bundleservice.BundleError):
+        bundleservice.loads('not json at all')
+    with pytest.raises(bundleservice.BundleError):
+        bundleservice.loads(json.dumps({'format': 'shopdb-guide', 'version': 1}))
+
+
+def test_a_bundle_from_a_newer_site_is_refused_rather_than_half_read():
+    newer = json.dumps({'format': 'shopdb-guide', 'version': 99,
+                        'slug': 'x', 'blocks': []})
+    with pytest.raises(bundleservice.BundleError) as error:
+        bundleservice.loads(newer)
+    assert 'newer than this site can read' in str(error.value)
+
+
+def test_an_unreadable_image_names_itself():
+    broken = json.dumps({
+        'format': 'shopdb-guide', 'version': 1, 'slug': 'x', 'blocks': [],
+        'images': [{'token': 'img001', 'mimetype': 'image/png',
+                    'filename': 'x.png', 'data': 'not base64 !!'}]})
+    with pytest.raises(bundleservice.BundleError) as error:
+        bundleservice.loads(broken)
+    assert 'img001' in str(error.value)