8953ce8210
It struck the heading and left everything underneath looking untouched, which is the opposite of what somebody ticking "this section is done" means. The strike now covers every block under that heading, down to the next heading of the same or higher rank - so a top-level section takes its sub-sections with it. That needed the rank, and the HTML importer was throwing it away: h2, h3 and h4 all arrived as a flat 'heading', so a parent heading had no children to cross out. It was the visible half of a real loss - a converted article kept its words and lost its shape. Headings now carry their level and render at it, and a guide written before this reads as flat, exactly as it did. Code blocks, tables and images inside a finished section fade but are not struck. A line through a command makes it unreadable, and re-reading the command is usually the reason to look back at a step already done. The five imported guides were re-imported so they carry their structure.
470 lines
17 KiB
Python
470 lines
17 KiB
Python
"""Turn a hand-written HTML article into guide blocks.
|
|
|
|
WHY THIS EXISTS
|
|
|
|
Sites arrive with a folder of KB articles: a procedure written once as a
|
|
styled HTML page, mailed around, opened from a share. Those pages are what a
|
|
guide is for, and retyping one into the editor is how a good procedure stays
|
|
on a share forever. This reads the page and produces blocks.
|
|
|
|
WHAT IT IS NOT
|
|
|
|
It is not an HTML renderer, and it deliberately loses things. Colours, fonts,
|
|
widths, nested layout and any decoration go, because that is the entire
|
|
point of storing blocks rather than markup - the theme decides how a guide
|
|
looks, and pasted styling is what makes a document unreadable in dark mode.
|
|
A converted guide is a DRAFT: somebody reads it before it is published.
|
|
|
|
WHAT IT RECOGNISES
|
|
|
|
The GE Aerospace KB template's own vocabulary, which is also the shape most
|
|
hand-written procedure pages take:
|
|
|
|
div.step + span.step-number -> step (guides do the numbering)
|
|
div.alert alert-<variant> -> callout
|
|
div.code-block, pre, code -> code
|
|
ul.checkbox-list -> checklist (tickable, and remembered)
|
|
h1..h4 -> heading (h1 becomes the title)
|
|
p -> paragraph
|
|
ul / ol -> bullets / numbers
|
|
table -> table
|
|
img, inline svg -> image (returned for upload)
|
|
|
|
Anything else contributes its text. A div wrapping other blocks is walked
|
|
through, not turned into one - guides have no nesting.
|
|
|
|
INLINE TEXT is the restricted Markdown subset the renderer accepts: bold,
|
|
italic, inline code, links, line breaks. Anything else becomes plain text,
|
|
which is a loss the format intends.
|
|
"""
|
|
|
|
import base64
|
|
import re
|
|
from html import unescape
|
|
from html.parser import HTMLParser
|
|
from urllib.parse import unquote
|
|
|
|
# Blocks a guide has no concept of, and page furniture no reader wants twice.
|
|
# The nav is a table of contents the guide generates from its own headings.
|
|
SKIPCLASSES = ('header', 'header-content', 'header-logo', 'header-subtitle',
|
|
'kb-badge', 'nav', 'footer', 'footer-content', 'copy-button')
|
|
SKIPTAGS = ('head', 'style', 'script', 'button', 'noscript')
|
|
|
|
VOIDTAGS = ('br', 'img', 'meta', 'link', 'input', 'hr', 'source', 'col',
|
|
'area', 'base', 'embed', 'track', 'wbr')
|
|
|
|
# The renderer knows info, warning and success. A danger alert is a warning
|
|
# with a louder colour, and colour is not something a block carries.
|
|
CALLOUTVARIANTS = {'alert-info': 'info', 'alert-warning': 'warning',
|
|
'alert-success': 'success', 'alert-danger': 'warning',
|
|
'alert-error': 'warning'}
|
|
|
|
DATAURI = re.compile(r'^data:([\w.+-]+/[\w.+-]+)?(;charset=[\w-]+)?(;base64)?,(.*)$',
|
|
re.IGNORECASE | re.DOTALL)
|
|
|
|
|
|
class Node:
|
|
"""One element or one run of text. A tree small enough to walk twice."""
|
|
|
|
__slots__ = ('tag', 'attrs', 'children', 'text')
|
|
|
|
def __init__(self, tag, attrs=None, text=None):
|
|
self.tag = tag
|
|
self.attrs = attrs or {}
|
|
self.children = []
|
|
self.text = text
|
|
|
|
@property
|
|
def classes(self):
|
|
return (self.attrs.get('class') or '').split()
|
|
|
|
|
|
class _Tree(HTMLParser):
|
|
"""Builds the node tree. Tolerant: real articles have unclosed tags."""
|
|
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.root = Node('#root')
|
|
self.stack = [self.root]
|
|
|
|
# A <p> that is never closed is normal HTML, and the browser closes it the
|
|
# moment a block element starts. Without the same rule, one unclosed <p>
|
|
# swallows the entire rest of the article into a single paragraph.
|
|
CLOSESPARAGRAPH = ('p', 'div', 'section', 'article', 'ul', 'ol', 'li',
|
|
'table', 'tr', 'td', 'th', 'pre', 'blockquote', 'hr',
|
|
'h1', 'h2', 'h3', 'h4', 'h5', 'h6')
|
|
|
|
def _autoclose(self, tag):
|
|
openers = {'li': ('li',), 'tr': ('tr', 'td', 'th'),
|
|
'td': ('td', 'th'), 'th': ('td', 'th')}.get(tag, ())
|
|
if tag in self.CLOSESPARAGRAPH:
|
|
openers = openers + ('p',)
|
|
while len(self.stack) > 1 and self.stack[-1].tag in openers:
|
|
self.stack.pop()
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self._autoclose(tag)
|
|
node = Node(tag, {k: (v or '') for k, v in attrs})
|
|
self.stack[-1].children.append(node)
|
|
if tag not in VOIDTAGS:
|
|
self.stack.append(node)
|
|
|
|
def handle_startendtag(self, tag, attrs):
|
|
self.stack[-1].children.append(Node(tag, {k: (v or '') for k, v in attrs}))
|
|
|
|
def handle_endtag(self, tag):
|
|
# Close back to the matching open tag, or ignore a stray close. An
|
|
# article with one unclosed <div> must still import.
|
|
for depth in range(len(self.stack) - 1, 0, -1):
|
|
if self.stack[depth].tag == tag:
|
|
del self.stack[depth:]
|
|
return
|
|
|
|
def handle_data(self, data):
|
|
if data.strip():
|
|
self.stack[-1].children.append(Node('#text', text=data))
|
|
|
|
|
|
def _skip(node):
|
|
if node.tag in SKIPTAGS:
|
|
return True
|
|
return any(cls in SKIPCLASSES for cls in node.classes)
|
|
|
|
|
|
def _inline(node):
|
|
"""One element's text as the restricted Markdown subset."""
|
|
out = []
|
|
for child in node.children:
|
|
if child.tag == '#text':
|
|
out.append(re.sub(r'\s+', ' ', child.text))
|
|
elif _skip(child):
|
|
continue
|
|
elif child.tag == 'br':
|
|
out.append('\n')
|
|
elif child.tag in ('strong', 'b'):
|
|
inner = _inline(child).strip()
|
|
out.append('**{}**'.format(inner) if inner else '')
|
|
elif child.tag in ('em', 'i'):
|
|
inner = _inline(child).strip()
|
|
out.append('*{}*'.format(inner) if inner else '')
|
|
elif child.tag in ('code', 'kbd', 'samp', 'tt'):
|
|
inner = _inline(child).strip()
|
|
out.append('`{}`'.format(inner) if inner else '')
|
|
elif child.tag == 'a':
|
|
inner = _inline(child).strip()
|
|
href = child.attrs.get('href', '').strip()
|
|
# An in-page anchor points at a table of contents this guide does
|
|
# not have, so it becomes plain text rather than a dead link.
|
|
if href and not href.startswith('#'):
|
|
out.append('[{}]({})'.format(inner, href))
|
|
else:
|
|
out.append(inner)
|
|
else:
|
|
# span.command-inline and friends: the KB template marks up inline
|
|
# commands with a class rather than <code>, so honour that too.
|
|
inner = _inline(child)
|
|
if child.tag == 'span' and 'command-inline' in child.classes:
|
|
stripped = inner.strip()
|
|
out.append('`{}`'.format(stripped) if stripped else '')
|
|
else:
|
|
out.append(inner)
|
|
text = ''.join(out)
|
|
text = re.sub(r'[ \t]+', ' ', text)
|
|
return text.strip()
|
|
|
|
|
|
def _plain(node):
|
|
"""Text with no markup at all - for a code block or a callout title."""
|
|
if node.tag == '#text':
|
|
return node.text
|
|
if _skip(node):
|
|
return ''
|
|
return ''.join(_plain(child) for child in node.children)
|
|
|
|
|
|
def _imagepayload(node):
|
|
"""(mimetype, bytes, filename) for an <img>, or None if it is not embeddable."""
|
|
src = unescape(node.attrs.get('src', '').strip())
|
|
match = DATAURI.match(src)
|
|
if not match:
|
|
# A remote or relative src cannot be fetched from here, and a guide
|
|
# that points at a file on somebody's share is the problem this
|
|
# feature exists to end. Reported, not imported.
|
|
return None
|
|
mimetype = (match.group(1) or 'image/png').lower()
|
|
payload = match.group(4)
|
|
try:
|
|
data = (base64.b64decode(payload, validate=False) if match.group(3)
|
|
else unquote(payload).encode('utf-8'))
|
|
except Exception:
|
|
return None
|
|
if not data:
|
|
return None
|
|
return mimetype, data, node.attrs.get('alt') or 'image'
|
|
|
|
|
|
class _Builder:
|
|
def __init__(self):
|
|
self.blocks = []
|
|
self.images = []
|
|
self.title = None
|
|
self.skipped = []
|
|
|
|
def _add(self, block):
|
|
block['id'] = 'b{:03d}'.format(len(self.blocks) + 1)
|
|
self.blocks.append(block)
|
|
|
|
def _items(self, node, tag='li'):
|
|
items = []
|
|
for child in node.children:
|
|
if child.tag != tag or _skip(child):
|
|
continue
|
|
text = _inline(child)
|
|
if text:
|
|
items.append({'id': 'i{:03d}'.format(len(items) + 1 + len(self.blocks) * 100),
|
|
'text': text})
|
|
return items
|
|
|
|
def walk(self, node):
|
|
for child in node.children:
|
|
if child.tag == '#text' or _skip(child):
|
|
continue
|
|
self.visit(child)
|
|
|
|
def visit(self, node):
|
|
tag = node.tag
|
|
classes = node.classes
|
|
|
|
if tag == 'h1':
|
|
# The article's own title, not a heading inside it.
|
|
if self.title is None:
|
|
self.title = _inline(node)
|
|
return
|
|
|
|
if tag in ('h2', 'h3', 'h4', 'h5'):
|
|
text = _inline(node)
|
|
if text:
|
|
# The LEVEL travels with it. Without it every heading is a
|
|
# sibling, and a reader who ticks a section off crosses out its
|
|
# title and nothing underneath - the sub-headings that belong to
|
|
# it look untouched, because nothing recorded that they belong.
|
|
self._add({'type': 'heading', 'level': int(tag[1]),
|
|
'text': text})
|
|
return
|
|
|
|
if tag == 'p':
|
|
text = _inline(node)
|
|
if text:
|
|
self._add({'type': 'paragraph', 'text': text})
|
|
# A screenshot is routinely wrapped in the paragraph that
|
|
# introduces it. Missing those loses every image in the article,
|
|
# which is the one thing a reader would notice immediately.
|
|
for image in _descendants(node, 'img'):
|
|
self._image(image)
|
|
return
|
|
|
|
if tag in ('pre',) or 'code-block' in classes:
|
|
text = _plain(node).strip('\n')
|
|
text = '\n'.join(line.rstrip() for line in text.splitlines())
|
|
if text.strip():
|
|
self._add({'type': 'code', 'text': text})
|
|
return
|
|
|
|
if 'step' in classes:
|
|
self._step(node)
|
|
return
|
|
|
|
if any(cls.startswith('alert') for cls in classes):
|
|
self._callout(node, classes)
|
|
return
|
|
|
|
if tag == 'ul':
|
|
items = self._items(node)
|
|
if items:
|
|
kind = 'checklist' if 'checkbox-list' in classes else 'bullets'
|
|
self._add({'type': kind, 'items': items})
|
|
return
|
|
|
|
if tag == 'ol':
|
|
items = self._items(node)
|
|
if items:
|
|
self._add({'type': 'numbers', 'items': items})
|
|
return
|
|
|
|
if tag == 'table':
|
|
self._table(node)
|
|
return
|
|
|
|
if tag == 'img':
|
|
self._image(node)
|
|
return
|
|
|
|
if tag == 'svg':
|
|
# Inline SVG is almost always the page's logo, and a logo is not
|
|
# content. Named in the report so nothing disappears silently.
|
|
self.skipped.append('inline <svg> dropped')
|
|
return
|
|
|
|
if tag in ('div', 'section', 'article', 'main', 'body', 'html',
|
|
'figure', 'blockquote', 'li', 'td', 'th', 'tr', 'tbody',
|
|
'thead', 'span', 'a', 'strong', 'em', 'small', 'label'):
|
|
# A wrapper. Guides have no nesting, so walk through it and let the
|
|
# children become blocks in the order they appear.
|
|
self.walk(node)
|
|
return
|
|
|
|
self.walk(node)
|
|
|
|
def _step(self, node):
|
|
"""div.step: a numbered step, whose body may hold other blocks."""
|
|
title = ''
|
|
text = []
|
|
rest = []
|
|
for child in node.children:
|
|
if child.tag == '#text' or _skip(child):
|
|
continue
|
|
if child.tag == 'span' and 'step-number' in child.classes:
|
|
continue # guides number their own steps
|
|
if child.tag in ('h3', 'h4', 'h5') and not title:
|
|
title = _inline(child)
|
|
continue
|
|
if child.tag == 'p' and not rest:
|
|
paragraph = _inline(child)
|
|
if paragraph:
|
|
text.append(paragraph)
|
|
continue
|
|
rest.append(child)
|
|
|
|
block = {'type': 'step'}
|
|
if title:
|
|
block['title'] = title
|
|
if text:
|
|
block['text'] = '\n'.join(text)
|
|
self._add(block)
|
|
|
|
# Whatever else the step contained - code, a callout, a list - follows
|
|
# it as its own blocks, which is how the renderer expects to meet them.
|
|
for child in rest:
|
|
self.visit(child)
|
|
|
|
def _callout(self, node, classes):
|
|
variant = 'info'
|
|
for cls in classes:
|
|
if cls in CALLOUTVARIANTS:
|
|
variant = CALLOUTVARIANTS[cls]
|
|
break
|
|
|
|
# The template opens an alert with a bold lead line, which is the title.
|
|
title = ''
|
|
body = []
|
|
for child in node.children:
|
|
if child.tag == '#text':
|
|
piece = re.sub(r'\s+', ' ', child.text).strip()
|
|
if piece:
|
|
body.append(piece)
|
|
elif _skip(child):
|
|
continue
|
|
elif child.tag in ('strong', 'b') and not title and not body:
|
|
title = _plain(child).strip()
|
|
else:
|
|
piece = _inline(child)
|
|
if piece:
|
|
body.append(piece)
|
|
|
|
block = {'type': 'callout', 'variant': variant, 'text': ' '.join(body)}
|
|
if title:
|
|
block['title'] = title
|
|
if block['text'] or title:
|
|
self._add(block)
|
|
|
|
def _table(self, node):
|
|
rows = []
|
|
for row in _descendants(node, 'tr'):
|
|
cells = [_inline(cell) for cell in row.children
|
|
if cell.tag in ('td', 'th')]
|
|
if cells:
|
|
rows.append((cells, any(cell.tag == 'th' for cell in row.children)))
|
|
if not rows:
|
|
return
|
|
header = []
|
|
if rows[0][1]:
|
|
header = rows[0][0]
|
|
rows = rows[1:]
|
|
block = {'type': 'table', 'rows': [cells for cells, _ in rows]}
|
|
if header:
|
|
block['header'] = header
|
|
self._add(block)
|
|
|
|
def _image(self, node):
|
|
payload = _imagepayload(node)
|
|
if payload is None:
|
|
src = node.attrs.get('src', '')[:60]
|
|
self.skipped.append('image not embedded, skipped: {}'.format(src))
|
|
return
|
|
mimetype, data, alt = payload
|
|
token = 'img{:03d}'.format(len(self.images) + 1)
|
|
self.images.append({'token': token, 'mimetype': mimetype, 'data': data,
|
|
'filename': '{}{}'.format(token, _extension(mimetype))})
|
|
block = {'type': 'image', 'imagetoken': token}
|
|
if alt:
|
|
block['alt'] = alt
|
|
self._add(block)
|
|
|
|
|
|
def _descendants(node, tag):
|
|
for child in node.children:
|
|
if child.tag == tag:
|
|
yield child
|
|
else:
|
|
for found in _descendants(child, tag):
|
|
yield found
|
|
|
|
|
|
def _extension(mimetype):
|
|
return {'image/png': '.png', 'image/jpeg': '.jpg', 'image/gif': '.gif',
|
|
'image/webp': '.webp', 'image/svg+xml': '.svg'}.get(mimetype, '')
|
|
|
|
|
|
def _documenttitle(tree):
|
|
"""The <title>, minus the house suffix every KB page carries."""
|
|
for node in _descendants(tree, 'title'):
|
|
text = _plain(node).strip()
|
|
return re.sub(r'\s*[-|]\s*GE Aerospace Knowledge Base\s*$', '', text).strip()
|
|
return ''
|
|
|
|
|
|
def parse(html):
|
|
"""Parse an article. Returns a dict, raising nothing on malformed markup.
|
|
|
|
{
|
|
'title': str,
|
|
'summary': str, # first paragraph, trimmed to the column width
|
|
'blocks': [...], # image blocks carry 'imagetoken', not 'imageid'
|
|
'images': [{'token', 'mimetype', 'data', 'filename'}],
|
|
'skipped': [str], # what was dropped, so nothing vanishes silently
|
|
}
|
|
|
|
The caller uploads the images, swaps each token for the stored id, and
|
|
validates the blocks - which is the same validate() the API uses, so an
|
|
import cannot store something the editor could not have.
|
|
"""
|
|
tree = _Tree()
|
|
tree.feed(html)
|
|
tree.close()
|
|
|
|
builder = _Builder()
|
|
builder.walk(tree.root)
|
|
|
|
title = builder.title or _documenttitle(tree.root) or 'Untitled'
|
|
|
|
summary = ''
|
|
for block in builder.blocks:
|
|
if block['type'] == 'paragraph':
|
|
summary = re.sub(r'[*`]', '', block['text'])
|
|
break
|
|
if len(summary) > 300:
|
|
summary = summary[:297].rsplit(' ', 1)[0] + '...'
|
|
|
|
return {'title': title, 'summary': summary, 'blocks': builder.blocks,
|
|
'images': builder.images, 'skipped': builder.skipped}
|