guides export/import bundles

plugins/guides/cli.py

@@ -1,15 +1,22 @@
-"""CLI: bring an existing HTML article in as a guide.
+"""CLI: move guides in and out of a site.
 
-    flask guides import-html PATH [PATH ...]
+    flask guides import-html PATH [PATH ...]   an HTML article becomes a guide
+    flask guides export --out DIR              guides become portable files
+    flask guides import PATH [PATH ...]        those files become guides here
 
 A site that adopts this app usually arrives with a folder of procedures already
 written - styled HTML pages that get mailed around and opened off a share. This
 imports them so they live where they are used, become searchable, tickable and
 printable, and stop being a file somebody has to find.
 
-Imports land as DRAFTS, always. The conversion is faithful but it is not a
-render: layout, colour and anything decorative are dropped on purpose, and a
-person should read the result before anybody is told to follow it.
+Imports land as DRAFTS, always - both kinds. An HTML conversion is faithful but
+it is not a render, and a bundle written on another site has not been read by
+anybody here. Publishing is a decision somebody makes after reading.
+
+Sites are separate databases and the useful ones are air-gapped from where a
+guide gets written, so export/import is how the same procedure reaches all of
+them: one JSON file per guide, images carried inside it, moved by whatever
+moves files to that site.
 """
 
 import click
@@ -18,7 +25,7 @@ from flask.cli import AppGroup, with_appcontext
 from shopdb.api import db
 
 from .models import Guide, GuideImage
-from .services import authoring
+from .services import authoring, bundle as bundleservice
 from .services.blocks import BlockError
 from .services.htmlimport import parse
 
@@ -107,6 +114,117 @@ def build_cli():
             click.echo('  imported as /guides/{} (draft, revision {})'
                        .format(guide.slug, guide.revision))
 
+    @guides_cli.command('export')
+    @click.option('--out', 'outdir', required=True,
+                  type=click.Path(file_okay=False),
+                  help='Directory to write the bundles into. Created if needed.')
+    @click.option('--slug', 'slugs', multiple=True,
+                  help='Export just this guide. Repeatable. Default: all.')
+    @click.option('--published-only', is_flag=True,
+                  help='Skip guides that have never been published.')
+    @with_appcontext
+    def export_guides(outdir, slugs, published_only):
+        """Write guides out as portable .guide.json files."""
+        import os
+
+        query = Guide.query
+        if slugs:
+            query = query.filter(Guide.slug.in_(slugs))
+        if published_only:
+            query = query.filter(Guide.publishedrevisionid.isnot(None))
+        guides = query.order_by(Guide.slug).all()
+
+        missing = set(slugs) - {guide.slug for guide in guides}
+        for slug in sorted(missing):
+            click.echo('  NOT FOUND {}'.format(slug))
+
+        if not guides:
+            click.echo('Nothing to export.')
+            return
+
+        os.makedirs(outdir, exist_ok=True)
+        for guide in guides:
+            images = GuideImage.query.filter_by(guideid=guide.guideid).all()
+            payload = bundleservice.build(guide, images)
+            path = os.path.join(outdir, '{}.guide.json'.format(guide.slug))
+            with open(path, 'w', encoding='utf-8') as handle:
+                handle.write(bundleservice.dumps(payload))
+            click.echo('  {}  {} blocks, {} image(s), {:.0f} KB{}'.format(
+                os.path.basename(path), len(payload['blocks']),
+                len(payload['images']), os.path.getsize(path) / 1024,
+                '' if guide.publishedrevisionid else '  [draft here]'))
+        click.echo('{} guide(s) -> {}'.format(len(guides), outdir))
+
+    @guides_cli.command('import')
+    @click.argument('paths', nargs=-1, required=True,
+                    type=click.Path(exists=True))
+    @click.option('--userid', type=int, default=None,
+                  help='Record this user as the author on this site.')
+    @click.option('--replace', is_flag=True,
+                  help='If a guide already has the bundle\'s slug, edit it in '
+                       'place (a new revision) instead of creating a second '
+                       'one. This is how an updated guide is re-shipped.')
+    @click.option('--apply/--dry-run', default=False,
+                  help='Write to the database. The default reports only.')
+    @with_appcontext
+    def import_guides(paths, userid, replace, apply):
+        """Import .guide.json bundle(s) written by `guides export`."""
+        import os
+
+        files = []
+        for path in paths:
+            if os.path.isdir(path):
+                files.extend(sorted(os.path.join(path, name)
+                                    for name in os.listdir(path)
+                                    if name.endswith('.guide.json')))
+            else:
+                files.append(path)
+        if not files:
+            click.echo('No .guide.json files found.')
+            return
+
+        for path in files:
+            click.echo(os.path.basename(path))
+            try:
+                with open(path, encoding='utf-8') as handle:
+                    payload = bundleservice.loads(handle.read())
+            except bundleservice.BundleError as error:
+                click.echo('  SKIPPED {}'.format(error))
+                continue
+
+            existing = Guide.query.filter(Guide.slug == payload['slug']).first()
+            click.echo('  title   {}'.format(payload.get('title')))
+            click.echo('  blocks  {}, images {}'.format(
+                len(payload['blocks']), len(payload['images'])))
+            if existing is not None:
+                click.echo('  {} already has this slug{}'.format(
+                    'guide {}'.format(existing.guideid),
+                    ' - will be updated in place' if replace
+                    else ' - a SECOND guide will be created; pass --replace to '
+                         'update it instead'))
+
+            rejected = [image for image in payload['images']
+                        if image['mimetype'] not in ALLOWEDIMAGETYPES
+                        or len(image['data']) > MAXIMAGEBYTES]
+            for image in rejected:
+                click.echo('  DROPPED {} ({}) - not an accepted image'
+                           .format(image['token'], image['mimetype']))
+
+            if not apply:
+                click.echo('  dry run, nothing written. Pass --apply to import.')
+                continue
+
+            try:
+                guide = _importbundle(payload, userid, replace, rejected)
+            except BlockError as error:
+                db.session.rollback()
+                click.echo('  FAILED  {}'.format(error))
+                continue
+
+            db.session.commit()
+            click.echo('  imported as /guides/{} (draft, revision {})'
+                       .format(guide.slug, guide.revision))
+
     return [guides_cli]
 
 
@@ -156,3 +274,59 @@ def _import(parsed, name, tags, userid, replace, rejected):
                    tags=tags, blocks=blocks,
                    note='Imported from HTML', userid=userid)
     return guide
+
+
+def _importbundle(payload, userid, replace, rejected):
+    """Create or update a guide from a bundle. Caller commits."""
+    guide = None
+    if replace:
+        guide = Guide.query.filter(Guide.slug == payload['slug']).first()
+
+    if guide is None:
+        # The bundle's own slug when it is free, because the slug is the name
+        # somebody pastes into a work order and the same guide should have the
+        # same address on every site. A clash only suffixes.
+        slug = payload['slug']
+        if Guide.query.filter(Guide.slug == slug).first() is not None:
+            slug = _slugify(payload.get('title') or slug)
+        guide = Guide(title=payload.get('title') or slug, slug=slug,
+                      status='draft', createdby=userid)
+        db.session.add(guide)
+        db.session.flush()
+
+    # Replacing means the incoming bundle is the whole truth for this guide.
+    # Its blocks will reference the images arriving now, so the ones already
+    # stored become orphans - and re-shipping a guide monthly would otherwise
+    # grow a pile of them nothing points at.
+    if replace:
+        GuideImage.query.filter_by(guideid=guide.guideid).delete()
+        db.session.flush()
+
+    droppedtokens = {image['token'] for image in rejected}
+    ids = {}
+    for image in payload['images']:
+        if image['token'] in droppedtokens:
+            continue
+        row = GuideImage(guideid=guide.guideid, filename=image['filename'],
+                         mimetype=image['mimetype'],
+                         bytesize=len(image['data']), data=image['data'],
+                         uploadedby=userid)
+        db.session.add(row)
+        db.session.flush()
+        ids[image['token']] = row.guideimageid
+
+    blocks = []
+    for block in payload['blocks']:
+        block = dict(block)
+        token = block.pop('imagetoken', None)
+        if token is not None:
+            if token not in ids:
+                continue
+            block['imageid'] = ids[token]
+        blocks.append(block)
+
+    authoring.save(guide, title=payload.get('title'),
+                   summary=payload.get('summary'), tags=payload.get('tags'),
+                   blocks=blocks, note='Imported from a guide bundle',
+                   userid=userid)
+    return guide

plugins/guides/services/bundle.py

@@ -0,0 +1,130 @@
+"""A guide as one portable file.
+
+WHY
+
+    Sites are separate installations with separate databases, and the useful
+    ones are air-gapped from where a guide gets written. A procedure that can
+    only exist on the machine it was typed on is the problem this plugin was
+    built to end, one layer up: instead of a document stranded on a share, a
+    guide stranded in one site's database.
+
+    So: export to a single JSON file, carry it however files get carried
+    there - a share, a robocopy, a USB drive - and import it on the other side.
+
+WHAT TRAVELS
+
+    Title, summary, tags and the blocks, plus every image the blocks reference,
+    inline as base64. One file, no companion directory: half a guide arriving
+    is worse than none, and a bundle that needs its images to be copied
+    alongside it is a bundle that will eventually arrive without them.
+
+WHAT DOES NOT TRAVEL
+
+    Ids, revision history, authorship and publication state. A guide's id and
+    its revision numbering belong to the database that issued them, and an
+    imported guide starts as a draft on the far side, at revision 1, owned by
+    whoever ran the import. The SLUG travels, because the slug is the name
+    somebody pastes into a work order and the whole point is that the same
+    guide is the same guide on both sites.
+"""
+
+import base64
+import json
+
+FORMAT = 'shopdb-guide'
+VERSION = 1
+
+# An image block points at a stored image by its id, and an id means nothing on
+# the far side. Blocks travel referring to a token instead, and the importer
+# swaps the tokens for the ids its own database issued.
+IMAGETOKEN = 'imagetoken'
+IMAGEID = 'imageid'
+
+
+class BundleError(ValueError):
+    """The file is not a guide bundle, or is one this version cannot read."""
+
+
+def build(guide, images):
+    """One guide plus its images as a plain dict, ready to be written as JSON.
+
+    `images` is the GuideImage rows for this guide. Only the ones its blocks
+    actually reference are carried - an orphaned paste from a draft is not part
+    of the procedure and does not need to cross the gap.
+    """
+    byid = {image.guideimageid: image for image in images}
+    used = []
+    tokens = {}
+    blocks = []
+    for block in guide.blocks:
+        block = dict(block)
+        imageid = block.pop(IMAGEID, None)
+        if imageid is not None:
+            image = byid.get(imageid)
+            if image is None:
+                # The image is gone. Carrying a block that points at nothing
+                # would render as "Image unavailable" on the far side, which
+                # reads as an import failure rather than as missing data.
+                continue
+            if imageid not in tokens:
+                tokens[imageid] = 'img{:03d}'.format(len(used) + 1)
+                used.append(image)
+            block[IMAGETOKEN] = tokens[imageid]
+        blocks.append(block)
+
+    return {
+        'format': FORMAT,
+        'version': VERSION,
+        'slug': guide.slug,
+        'title': guide.title,
+        'summary': guide.summary,
+        'tags': guide.tags,
+        'blocks': blocks,
+        'images': [{
+            'token': tokens[image.guideimageid],
+            'filename': image.filename,
+            'mimetype': image.mimetype,
+            'data': base64.b64encode(image.data).decode('ascii'),
+        } for image in used],
+    }
+
+
+def dumps(bundle):
+    return json.dumps(bundle, indent=2, sort_keys=False) + '\n'
+
+
+def loads(text):
+    """Parse a bundle, with the image data decoded back to bytes.
+
+    Refuses anything that is not this format. A JSON file that happens to be
+    somewhere near the others must not be half-imported before the failure
+    shows up.
+    """
+    try:
+        bundle = json.loads(text)
+    except ValueError as error:
+        raise BundleError('not valid JSON: {}'.format(error))
+
+    if not isinstance(bundle, dict) or bundle.get('format') != FORMAT:
+        raise BundleError('not a {} file'.format(FORMAT))
+    version = bundle.get('version')
+    if not isinstance(version, int) or version > VERSION:
+        raise BundleError(
+            'bundle version {} is newer than this site can read (max {})'
+            .format(version, VERSION))
+    if not isinstance(bundle.get('blocks'), list):
+        raise BundleError('bundle has no blocks')
+
+    images = []
+    for image in bundle.get('images') or []:
+        try:
+            data = base64.b64decode(image['data'], validate=True)
+        except Exception:
+            raise BundleError('image {!r} is not readable'
+                              .format(image.get('token')))
+        images.append({'token': image.get('token'),
+                       'filename': image.get('filename'),
+                       'mimetype': (image.get('mimetype') or '').lower(),
+                       'data': data})
+    bundle['images'] = images
+    return bundle