470 lines
17 KiB
Python
470 lines
17 KiB
Python
"""Turn a hand-written HTML article into guide blocks.
|
|
|
|
WHY THIS EXISTS
|
|
|
|
Sites arrive with a folder of KB articles: a procedure written once as a
|
|
styled HTML page, mailed around, opened from a share. Those pages are what a
|
|
guide is for, and retyping one into the editor is how a good procedure stays
|
|
on a share forever. This reads the page and produces blocks.
|
|
|
|
WHAT IT IS NOT
|
|
|
|
It is not an HTML renderer, and it deliberately loses things. Colours, fonts,
|
|
widths, nested layout and any decoration go, because that is the entire
|
|
point of storing blocks rather than markup - the theme decides how a guide
|
|
looks, and pasted styling is what makes a document unreadable in dark mode.
|
|
A converted guide is a DRAFT: somebody reads it before it is published.
|
|
|
|
WHAT IT RECOGNISES
|
|
|
|
The GE Aerospace KB template's own vocabulary, which is also the shape most
|
|
hand-written procedure pages take:
|
|
|
|
div.step + span.step-number -> step (guides do the numbering)
|
|
div.alert alert-<variant> -> callout
|
|
div.code-block, pre, code -> code
|
|
ul.checkbox-list -> checklist (tickable, and remembered)
|
|
h1..h4 -> heading (h1 becomes the title)
|
|
p -> paragraph
|
|
ul / ol -> bullets / numbers
|
|
table -> table
|
|
img, inline svg -> image (returned for upload)
|
|
|
|
Anything else contributes its text. A div wrapping other blocks is walked
|
|
through, not turned into one - guides have no nesting.
|
|
|
|
INLINE TEXT is the restricted Markdown subset the renderer accepts: bold,
|
|
italic, inline code, links, line breaks. Anything else becomes plain text,
|
|
which is a loss the format intends.
|
|
"""
|
|
|
|
import base64
|
|
import re
|
|
from html import unescape
|
|
from html.parser import HTMLParser
|
|
from urllib.parse import unquote
|
|
|
|
# Blocks a guide has no concept of, and page furniture no reader wants twice.
|
|
# The nav is a table of contents the guide generates from its own headings.
|
|
SKIPCLASSES = ('header', 'header-content', 'header-logo', 'header-subtitle',
|
|
'kb-badge', 'nav', 'footer', 'footer-content', 'copy-button')
|
|
SKIPTAGS = ('head', 'style', 'script', 'button', 'noscript')
|
|
|
|
VOIDTAGS = ('br', 'img', 'meta', 'link', 'input', 'hr', 'source', 'col',
|
|
'area', 'base', 'embed', 'track', 'wbr')
|
|
|
|
# The renderer knows info, warning and success. A danger alert is a warning
|
|
# with a louder colour, and colour is not something a block carries.
|
|
CALLOUTVARIANTS = {'alert-info': 'info', 'alert-warning': 'warning',
|
|
'alert-success': 'success', 'alert-danger': 'warning',
|
|
'alert-error': 'warning'}
|
|
|
|
DATAURI = re.compile(r'^data:([\w.+-]+/[\w.+-]+)?(;charset=[\w-]+)?(;base64)?,(.*)$',
|
|
re.IGNORECASE | re.DOTALL)
|
|
|
|
|
|
class Node:
|
|
"""One element or one run of text. A tree small enough to walk twice."""
|
|
|
|
__slots__ = ('tag', 'attrs', 'children', 'text')
|
|
|
|
def __init__(self, tag, attrs=None, text=None):
|
|
self.tag = tag
|
|
self.attrs = attrs or {}
|
|
self.children = []
|
|
self.text = text
|
|
|
|
@property
|
|
def classes(self):
|
|
return (self.attrs.get('class') or '').split()
|
|
|
|
|
|
class _Tree(HTMLParser):
|
|
"""Builds the node tree. Tolerant: real articles have unclosed tags."""
|
|
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.root = Node('#root')
|
|
self.stack = [self.root]
|
|
|
|
# A <p> that is never closed is normal HTML, and the browser closes it the
|
|
# moment a block element starts. Without the same rule, one unclosed <p>
|
|
# swallows the entire rest of the article into a single paragraph.
|
|
CLOSESPARAGRAPH = ('p', 'div', 'section', 'article', 'ul', 'ol', 'li',
|
|
'table', 'tr', 'td', 'th', 'pre', 'blockquote', 'hr',
|
|
'h1', 'h2', 'h3', 'h4', 'h5', 'h6')
|
|
|
|
def _autoclose(self, tag):
|
|
openers = {'li': ('li',), 'tr': ('tr', 'td', 'th'),
|
|
'td': ('td', 'th'), 'th': ('td', 'th')}.get(tag, ())
|
|
if tag in self.CLOSESPARAGRAPH:
|
|
openers = openers + ('p',)
|
|
while len(self.stack) > 1 and self.stack[-1].tag in openers:
|
|
self.stack.pop()
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self._autoclose(tag)
|
|
node = Node(tag, {k: (v or '') for k, v in attrs})
|
|
self.stack[-1].children.append(node)
|
|
if tag not in VOIDTAGS:
|
|
self.stack.append(node)
|
|
|
|
def handle_startendtag(self, tag, attrs):
|
|
self.stack[-1].children.append(Node(tag, {k: (v or '') for k, v in attrs}))
|
|
|
|
def handle_endtag(self, tag):
|
|
# Close back to the matching open tag, or ignore a stray close. An
|
|
# article with one unclosed <div> must still import.
|
|
for depth in range(len(self.stack) - 1, 0, -1):
|
|
if self.stack[depth].tag == tag:
|
|
del self.stack[depth:]
|
|
return
|
|
|
|
def handle_data(self, data):
|
|
if data.strip():
|
|
self.stack[-1].children.append(Node('#text', text=data))
|
|
|
|
|
|
def _skip(node):
|
|
if node.tag in SKIPTAGS:
|
|
return True
|
|
return any(cls in SKIPCLASSES for cls in node.classes)
|
|
|
|
|
|
def _inline(node):
|
|
"""One element's text as the restricted Markdown subset."""
|
|
out = []
|
|
for child in node.children:
|
|
if child.tag == '#text':
|
|
out.append(re.sub(r'\s+', ' ', child.text))
|
|
elif _skip(child):
|
|
continue
|
|
elif child.tag == 'br':
|
|
out.append('\n')
|
|
elif child.tag in ('strong', 'b'):
|
|
inner = _inline(child).strip()
|
|
out.append('**{}**'.format(inner) if inner else '')
|
|
elif child.tag in ('em', 'i'):
|
|
inner = _inline(child).strip()
|
|
out.append('*{}*'.format(inner) if inner else '')
|
|
elif child.tag in ('code', 'kbd', 'samp', 'tt'):
|
|
inner = _inline(child).strip()
|
|
out.append('`{}`'.format(inner) if inner else '')
|
|
elif child.tag == 'a':
|
|
inner = _inline(child).strip()
|
|
href = child.attrs.get('href', '').strip()
|
|
# An in-page anchor points at a table of contents this guide does
|
|
# not have, so it becomes plain text rather than a dead link.
|
|
if href and not href.startswith('#'):
|
|
out.append('[{}]({})'.format(inner, href))
|
|
else:
|
|
out.append(inner)
|
|
else:
|
|
# span.command-inline and friends: the KB template marks up inline
|
|
# commands with a class rather than <code>, so honour that too.
|
|
inner = _inline(child)
|
|
if child.tag == 'span' and 'command-inline' in child.classes:
|
|
stripped = inner.strip()
|
|
out.append('`{}`'.format(stripped) if stripped else '')
|
|
else:
|
|
out.append(inner)
|
|
text = ''.join(out)
|
|
text = re.sub(r'[ \t]+', ' ', text)
|
|
return text.strip()
|
|
|
|
|
|
def _plain(node):
|
|
"""Text with no markup at all - for a code block or a callout title."""
|
|
if node.tag == '#text':
|
|
return node.text
|
|
if _skip(node):
|
|
return ''
|
|
return ''.join(_plain(child) for child in node.children)
|
|
|
|
|
|
def _imagepayload(node):
|
|
"""(mimetype, bytes, filename) for an <img>, or None if it is not embeddable."""
|
|
src = unescape(node.attrs.get('src', '').strip())
|
|
match = DATAURI.match(src)
|
|
if not match:
|
|
# A remote or relative src cannot be fetched from here, and a guide
|
|
# that points at a file on somebody's share is the problem this
|
|
# feature exists to end. Reported, not imported.
|
|
return None
|
|
mimetype = (match.group(1) or 'image/png').lower()
|
|
payload = match.group(4)
|
|
try:
|
|
data = (base64.b64decode(payload, validate=False) if match.group(3)
|
|
else unquote(payload).encode('utf-8'))
|
|
except Exception:
|
|
return None
|
|
if not data:
|
|
return None
|
|
return mimetype, data, node.attrs.get('alt') or 'image'
|
|
|
|
|
|
class _Builder:
|
|
def __init__(self):
|
|
self.blocks = []
|
|
self.images = []
|
|
self.title = None
|
|
self.skipped = []
|
|
|
|
def _add(self, block):
|
|
block['id'] = 'b{:03d}'.format(len(self.blocks) + 1)
|
|
self.blocks.append(block)
|
|
|
|
def _items(self, node, tag='li'):
|
|
items = []
|
|
for child in node.children:
|
|
if child.tag != tag or _skip(child):
|
|
continue
|
|
text = _inline(child)
|
|
if text:
|
|
items.append({'id': 'i{:03d}'.format(len(items) + 1 + len(self.blocks) * 100),
|
|
'text': text})
|
|
return items
|
|
|
|
def walk(self, node):
|
|
for child in node.children:
|
|
if child.tag == '#text' or _skip(child):
|
|
continue
|
|
self.visit(child)
|
|
|
|
def visit(self, node):
|
|
tag = node.tag
|
|
classes = node.classes
|
|
|
|
if tag == 'h1':
|
|
# The article's own title, not a heading inside it.
|
|
if self.title is None:
|
|
self.title = _inline(node)
|
|
return
|
|
|
|
if tag in ('h2', 'h3', 'h4', 'h5'):
|
|
text = _inline(node)
|
|
if text:
|
|
# The LEVEL travels with it. Without it every heading is a
|
|
# sibling, and a reader who ticks a section off crosses out its
|
|
# title and nothing underneath - the sub-headings that belong to
|
|
# it look untouched, because nothing recorded that they belong.
|
|
self._add({'type': 'heading', 'level': int(tag[1]),
|
|
'text': text})
|
|
return
|
|
|
|
if tag == 'p':
|
|
text = _inline(node)
|
|
if text:
|
|
self._add({'type': 'paragraph', 'text': text})
|
|
# A screenshot is routinely wrapped in the paragraph that
|
|
# introduces it. Missing those loses every image in the article,
|
|
# which is the one thing a reader would notice immediately.
|
|
for image in _descendants(node, 'img'):
|
|
self._image(image)
|
|
return
|
|
|
|
if tag in ('pre',) or 'code-block' in classes:
|
|
text = _plain(node).strip('\n')
|
|
text = '\n'.join(line.rstrip() for line in text.splitlines())
|
|
if text.strip():
|
|
self._add({'type': 'code', 'text': text})
|
|
return
|
|
|
|
if 'step' in classes:
|
|
self._step(node)
|
|
return
|
|
|
|
if any(cls.startswith('alert') for cls in classes):
|
|
self._callout(node, classes)
|
|
return
|
|
|
|
if tag == 'ul':
|
|
items = self._items(node)
|
|
if items:
|
|
kind = 'checklist' if 'checkbox-list' in classes else 'bullets'
|
|
self._add({'type': kind, 'items': items})
|
|
return
|
|
|
|
if tag == 'ol':
|
|
items = self._items(node)
|
|
if items:
|
|
self._add({'type': 'numbers', 'items': items})
|
|
return
|
|
|
|
if tag == 'table':
|
|
self._table(node)
|
|
return
|
|
|
|
if tag == 'img':
|
|
self._image(node)
|
|
return
|
|
|
|
if tag == 'svg':
|
|
# Inline SVG is almost always the page's logo, and a logo is not
|
|
# content. Named in the report so nothing disappears silently.
|
|
self.skipped.append('inline <svg> dropped')
|
|
return
|
|
|
|
if tag in ('div', 'section', 'article', 'main', 'body', 'html',
|
|
'figure', 'blockquote', 'li', 'td', 'th', 'tr', 'tbody',
|
|
'thead', 'span', 'a', 'strong', 'em', 'small', 'label'):
|
|
# A wrapper. Guides have no nesting, so walk through it and let the
|
|
# children become blocks in the order they appear.
|
|
self.walk(node)
|
|
return
|
|
|
|
self.walk(node)
|
|
|
|
def _step(self, node):
|
|
"""div.step: a numbered step, whose body may hold other blocks."""
|
|
title = ''
|
|
text = []
|
|
rest = []
|
|
for child in node.children:
|
|
if child.tag == '#text' or _skip(child):
|
|
continue
|
|
if child.tag == 'span' and 'step-number' in child.classes:
|
|
continue # guides number their own steps
|
|
if child.tag in ('h3', 'h4', 'h5') and not title:
|
|
title = _inline(child)
|
|
continue
|
|
if child.tag == 'p' and not rest:
|
|
paragraph = _inline(child)
|
|
if paragraph:
|
|
text.append(paragraph)
|
|
continue
|
|
rest.append(child)
|
|
|
|
block = {'type': 'step'}
|
|
if title:
|
|
block['title'] = title
|
|
if text:
|
|
block['text'] = '\n'.join(text)
|
|
self._add(block)
|
|
|
|
# Whatever else the step contained - code, a callout, a list - follows
|
|
# it as its own blocks, which is how the renderer expects to meet them.
|
|
for child in rest:
|
|
self.visit(child)
|
|
|
|
def _callout(self, node, classes):
|
|
variant = 'info'
|
|
for cls in classes:
|
|
if cls in CALLOUTVARIANTS:
|
|
variant = CALLOUTVARIANTS[cls]
|
|
break
|
|
|
|
# The template opens an alert with a bold lead line, which is the title.
|
|
title = ''
|
|
body = []
|
|
for child in node.children:
|
|
if child.tag == '#text':
|
|
piece = re.sub(r'\s+', ' ', child.text).strip()
|
|
if piece:
|
|
body.append(piece)
|
|
elif _skip(child):
|
|
continue
|
|
elif child.tag in ('strong', 'b') and not title and not body:
|
|
title = _plain(child).strip()
|
|
else:
|
|
piece = _inline(child)
|
|
if piece:
|
|
body.append(piece)
|
|
|
|
block = {'type': 'callout', 'variant': variant, 'text': ' '.join(body)}
|
|
if title:
|
|
block['title'] = title
|
|
if block['text'] or title:
|
|
self._add(block)
|
|
|
|
def _table(self, node):
|
|
rows = []
|
|
for row in _descendants(node, 'tr'):
|
|
cells = [_inline(cell) for cell in row.children
|
|
if cell.tag in ('td', 'th')]
|
|
if cells:
|
|
rows.append((cells, any(cell.tag == 'th' for cell in row.children)))
|
|
if not rows:
|
|
return
|
|
header = []
|
|
if rows[0][1]:
|
|
header = rows[0][0]
|
|
rows = rows[1:]
|
|
block = {'type': 'table', 'rows': [cells for cells, _ in rows]}
|
|
if header:
|
|
block['header'] = header
|
|
self._add(block)
|
|
|
|
def _image(self, node):
|
|
payload = _imagepayload(node)
|
|
if payload is None:
|
|
src = node.attrs.get('src', '')[:60]
|
|
self.skipped.append('image not embedded, skipped: {}'.format(src))
|
|
return
|
|
mimetype, data, alt = payload
|
|
token = 'img{:03d}'.format(len(self.images) + 1)
|
|
self.images.append({'token': token, 'mimetype': mimetype, 'data': data,
|
|
'filename': '{}{}'.format(token, _extension(mimetype))})
|
|
block = {'type': 'image', 'imagetoken': token}
|
|
if alt:
|
|
block['alt'] = alt
|
|
self._add(block)
|
|
|
|
|
|
def _descendants(node, tag):
|
|
for child in node.children:
|
|
if child.tag == tag:
|
|
yield child
|
|
else:
|
|
for found in _descendants(child, tag):
|
|
yield found
|
|
|
|
|
|
def _extension(mimetype):
|
|
return {'image/png': '.png', 'image/jpeg': '.jpg', 'image/gif': '.gif',
|
|
'image/webp': '.webp', 'image/svg+xml': '.svg'}.get(mimetype, '')
|
|
|
|
|
|
def _documenttitle(tree):
|
|
"""The <title>, minus the house suffix every KB page carries."""
|
|
for node in _descendants(tree, 'title'):
|
|
text = _plain(node).strip()
|
|
return re.sub(r'\s*[-|]\s*GE Aerospace Knowledge Base\s*$', '', text).strip()
|
|
return ''
|
|
|
|
|
|
def parse(html):
|
|
"""Parse an article. Returns a dict, raising nothing on malformed markup.
|
|
|
|
{
|
|
'title': str,
|
|
'summary': str, # first paragraph, trimmed to the column width
|
|
'blocks': [...], # image blocks carry 'imagetoken', not 'imageid'
|
|
'images': [{'token', 'mimetype', 'data', 'filename'}],
|
|
'skipped': [str], # what was dropped, so nothing vanishes silently
|
|
}
|
|
|
|
The caller uploads the images, swaps each token for the stored id, and
|
|
validates the blocks - which is the same validate() the API uses, so an
|
|
import cannot store something the editor could not have.
|
|
"""
|
|
tree = _Tree()
|
|
tree.feed(html)
|
|
tree.close()
|
|
|
|
builder = _Builder()
|
|
builder.walk(tree.root)
|
|
|
|
title = builder.title or _documenttitle(tree.root) or 'Untitled'
|
|
|
|
summary = ''
|
|
for block in builder.blocks:
|
|
if block['type'] == 'paragraph':
|
|
summary = re.sub(r'[*`]', '', block['text'])
|
|
break
|
|
if len(summary) > 300:
|
|
summary = summary[:297].rsplit(' ', 1)[0] + '...'
|
|
|
|
return {'title': title, 'summary': summary, 'blocks': builder.blocks,
|
|
'images': builder.images, 'skipped': builder.skipped}
|