that is never closed is normal HTML, and the browser closes it the # moment a block element starts. Without the same rule, one unclosed
# swallows the entire rest of the article into a single paragraph. CLOSESPARAGRAPH = ('p', 'div', 'section', 'article', 'ul', 'ol', 'li', 'table', 'tr', 'td', 'th', 'pre', 'blockquote', 'hr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6') def _autoclose(self, tag): openers = {'li': ('li',), 'tr': ('tr', 'td', 'th'), 'td': ('td', 'th'), 'th': ('td', 'th')}.get(tag, ()) if tag in self.CLOSESPARAGRAPH: openers = openers + ('p',) while len(self.stack) > 1 and self.stack[-1].tag in openers: self.stack.pop() def handle_starttag(self, tag, attrs): self._autoclose(tag) node = Node(tag, {k: (v or '') for k, v in attrs}) self.stack[-1].children.append(node) if tag not in VOIDTAGS: self.stack.append(node) def handle_startendtag(self, tag, attrs): self.stack[-1].children.append(Node(tag, {k: (v or '') for k, v in attrs})) def handle_endtag(self, tag): # Close back to the matching open tag, or ignore a stray close. An # article with one unclosed
must still import.
for depth in range(len(self.stack) - 1, 0, -1):
if self.stack[depth].tag == tag:
del self.stack[depth:]
return
def handle_data(self, data):
if data.strip():
self.stack[-1].children.append(Node('#text', text=data))
def _skip(node):
if node.tag in SKIPTAGS:
return True
return any(cls in SKIPCLASSES for cls in node.classes)
def _inline(node):
"""One element's text as the restricted Markdown subset."""
out = []
for child in node.children:
if child.tag == '#text':
out.append(re.sub(r'\s+', ' ', child.text))
elif _skip(child):
continue
elif child.tag == 'br':
out.append('\n')
elif child.tag in ('strong', 'b'):
inner = _inline(child).strip()
out.append('**{}**'.format(inner) if inner else '')
elif child.tag in ('em', 'i'):
inner = _inline(child).strip()
out.append('*{}*'.format(inner) if inner else '')
elif child.tag in ('code', 'kbd', 'samp', 'tt'):
inner = _inline(child).strip()
out.append('`{}`'.format(inner) if inner else '')
elif child.tag == 'a':
inner = _inline(child).strip()
href = child.attrs.get('href', '').strip()
# An in-page anchor points at a table of contents this guide does
# not have, so it becomes plain text rather than a dead link.
if href and not href.startswith('#'):
out.append('[{}]({})'.format(inner, href))
else:
out.append(inner)
else:
# span.command-inline and friends: the KB template marks up inline
# commands with a class rather than
, so honour that too.
inner = _inline(child)
if child.tag == 'span' and 'command-inline' in child.classes:
stripped = inner.strip()
out.append('`{}`'.format(stripped) if stripped else '')
else:
out.append(inner)
text = ''.join(out)
text = re.sub(r'[ \t]+', ' ', text)
return text.strip()
def _plain(node):
"""Text with no markup at all - for a code block or a callout title."""
if node.tag == '#text':
return node.text
if _skip(node):
return ''
return ''.join(_plain(child) for child in node.children)
def _imagepayload(node):
"""(mimetype, bytes, filename) for an ![]()
, or None if it is not embeddable."""
src = unescape(node.attrs.get('src', '').strip())
match = DATAURI.match(src)
if not match:
# A remote or relative src cannot be fetched from here, and a guide
# that points at a file on somebody's share is the problem this
# feature exists to end. Reported, not imported.
return None
mimetype = (match.group(1) or 'image/png').lower()
payload = match.group(4)
try:
data = (base64.b64decode(payload, validate=False) if match.group(3)
else unquote(payload).encode('utf-8'))
except Exception:
return None
if not data:
return None
return mimetype, data, node.attrs.get('alt') or 'image'
class _Builder:
def __init__(self):
self.blocks = []
self.images = []
self.title = None
self.skipped = []
def _add(self, block):
block['id'] = 'b{:03d}'.format(len(self.blocks) + 1)
self.blocks.append(block)
def _items(self, node, tag='li'):
items = []
for child in node.children:
if child.tag != tag or _skip(child):
continue
text = _inline(child)
if text:
items.append({'id': 'i{:03d}'.format(len(items) + 1 + len(self.blocks) * 100),
'text': text})
return items
def walk(self, node):
for child in node.children:
if child.tag == '#text' or _skip(child):
continue
self.visit(child)
def visit(self, node):
tag = node.tag
classes = node.classes
if tag == 'h1':
# The article's own title, not a heading inside it.
if self.title is None:
self.title = _inline(node)
return
if tag in ('h2', 'h3', 'h4', 'h5'):
text = _inline(node)
if text:
# The LEVEL travels with it. Without it every heading is a
# sibling, and a reader who ticks a section off crosses out its
# title and nothing underneath - the sub-headings that belong to
# it look untouched, because nothing recorded that they belong.
self._add({'type': 'heading', 'level': int(tag[1]),
'text': text})
return
if tag == 'p':
text = _inline(node)
if text:
self._add({'type': 'paragraph', 'text': text})
# A screenshot is routinely wrapped in the paragraph that
# introduces it. Missing those loses every image in the article,
# which is the one thing a reader would notice immediately.
for image in _descendants(node, 'img'):
self._image(image)
return
if tag in ('pre',) or 'code-block' in classes:
text = _plain(node).strip('\n')
text = '\n'.join(line.rstrip() for line in text.splitlines())
if text.strip():
self._add({'type': 'code', 'text': text})
return
if 'step' in classes:
self._step(node)
return
if any(cls.startswith('alert') for cls in classes):
self._callout(node, classes)
return
if tag == 'ul':
items = self._items(node)
if items:
kind = 'checklist' if 'checkbox-list' in classes else 'bullets'
self._add({'type': kind, 'items': items})
return
if tag == 'ol':
items = self._items(node)
if items:
self._add({'type': 'numbers', 'items': items})
return
if tag == 'table':
self._table(node)
return
if tag == 'img':
self._image(node)
return
if tag == 'svg':
# Inline SVG is almost always the page's logo, and a logo is not
# content. Named in the report so nothing disappears silently.
self.skipped.append('inline