"""Unattend.xml parser, form-edit applier, linter and builder.

Round-trips a Windows ``FlatUnattendW10.xml`` between the disk format
and a flat dict the editor template can render. ``parse_unattend()``
reads disk + returns a dict; ``apply_form()`` writes a form dict back
INTO the existing file text (see "Form edits" below: only what changed
is touched, everything else stays byte for byte); ``lint_text()`` is
the webapp copy of scripts/lint-unattend.py; ``extract_form_data()``
converts a Flask ``request.form`` MultiDict into the form dict.
``build_unattend_xml()`` (a fresh file from a dict) is kept for callers
that want one, but no save path uses it any more: it dropped comments,
rewrote *arch* and re-flowed every attribute.
"""

import html
import os
import re

from lxml import etree

import config

UNATTEND_TEMPLATE = """\
<?xml version="1.0" encoding="utf-8"?>
<unattend xmlns="urn:schemas-microsoft-com:unattend"
          xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State">
  <settings pass="windowsPE" />
  <settings pass="offlineServicing" />
  <settings pass="specialize" />
  <settings pass="oobeSystem" />
</unattend>
"""


def qn(tag):
    """Return a tag qualified with the default unattend namespace."""
    return f"{{{config.UNATTEND_NS}}}{tag}"


def qwcm(attr):
    """Return an attribute qualified with the wcm namespace."""
    return f"{{{config.WCM_NS}}}{attr}"


def _find_or_create(parent, tag):
    """Find the first child with *tag* or create it."""
    el = parent.find(tag, namespaces={"": config.UNATTEND_NS})
    if el is None:
        el = etree.SubElement(parent, qn(tag.split("}")[-1]) if "}" not in tag else tag)
    return el


def _settings_pass(root, pass_name):
    """Return the <settings pass="..."> element, creating if needed."""
    for s in root.findall(qn("settings")):
        if s.get("pass") == pass_name:
            return s
    s = etree.SubElement(root, qn("settings"))
    s.set("pass", pass_name)
    return s


def _empty_data(defaults):
    """The dict parse_unattend returns. OOBE values: with defaults (no file
    yet) the settings a new answer file gets; for an existing file a
    setting that is not in it is "" (not set), so the form shows what the
    file holds and a save does not add settings nobody chose."""
    data = {
        "driver_paths": [],
        "computer_name": "",
        "registered_organization": "",
        "registered_owner": "",
        "time_zone": "",
        "specialize_commands": [],
        "oobe": {
            "HideEULAPage": "true",
            "HideOEMRegistrationScreen": "true",
            "HideOnlineAccountScreens": "true",
            "HideWirelessSetupInOOBE": "true",
            "HideLocalAccountScreen": "true",
            "NetworkLocation": "Work",
            "ProtectYourPC": "3",
            "SkipUserOOBE": "true",
            "SkipMachineOOBE": "true",
        },
        "firstlogon_commands": [],
        "user_accounts": [],
        "autologon": {
            "enabled": "",
            "username": "",
            "password": "",
            "plain_text": "true",
            "logon_count": "",
        },
        "intl": {
            "input_locale": "",
            "system_locale": "",
            "ui_language": "",
            "user_locale": "",
        },
        "oobe_timezone": "",
        "raw_xml": "",
    }
    if not defaults:
        data["oobe"] = {k: "" for k in data["oobe"]}
    return data


def _comment_text(c):
    return " ".join((c.text or "").split())


def _notes(el):
    """Comments that belong to a list item: the ones directly above it
    (up to the previous element) and the ones inside it. Shown read-only
    on the form; they move with the item (see Form edits)."""
    lead = []
    for sib in el.itersiblings(preceding=True):
        if not isinstance(sib, etree._Comment):
            break
        lead.append(_comment_text(sib))
    lead.reverse()
    inner = [_comment_text(c) for c in el.iter(etree.Comment)]
    return {"note": [t for t in lead if t], "inner_note": [t for t in inner if t]}


def parse_unattend(xml_path):
    """Parse an unattend.xml and return a dict of editable data."""
    if not os.path.isfile(xml_path):
        data = _empty_data(defaults=True)
        data["raw_xml"] = UNATTEND_TEMPLATE
        return data
    # errors="replace": a stray non-UTF-8 byte must not take the page down;
    # the save path refuses a form save on such a file (views/images_unattend).
    with open(xml_path, "r", encoding="utf-8", errors="replace") as fh:
        return parse_unattend_text(fh.read())


def parse_unattend_text(raw):
    """parse_unattend for text already read (a BOM, if any, is ignored)."""
    data = _empty_data(defaults=False)
    data["raw_xml"] = raw

    try:
        root = etree.fromstring(raw.lstrip("\ufeff").encode("utf-8"))
    except etree.XMLSyntaxError:
        return data

    ns = {"u": config.UNATTEND_NS}

    # --- offlineServicing: DriverPaths ---
    # By local name: the shopfloor file declares xmlns="" on
    # PathAndCredentials, which takes it (and Path) OUT of the unattend
    # namespace. A namespaced match missed it, and the old form save then
    # dropped W:\Drivers from that image.
    for dp_el in root.xpath(
        "u:settings[@pass='offlineServicing']//*[local-name()='PathAndCredentials']"
        "/*[local-name()='Path']",
        namespaces=ns,
    ):
        data["driver_paths"].append((dp_el.text or "").strip())

    # --- specialize: Shell-Setup ---
    for comp in root.xpath("u:settings[@pass='specialize']/u:component", namespaces=ns):
        comp_name = comp.get("name", "")
        if "Shell-Setup" in comp_name:
            for tag, key in [
                ("ComputerName", "computer_name"),
                ("RegisteredOrganization", "registered_organization"),
                ("RegisteredOwner", "registered_owner"),
                ("TimeZone", "time_zone"),
            ]:
                el = comp.find(qn(tag))
                if el is not None and el.text:
                    data[key] = el.text.strip()

    # --- specialize: RunSynchronous commands ---
    for cmd in root.xpath(
        "u:settings[@pass='specialize']//u:RunSynchronousCommand",
        namespaces=ns,
    ):
        order_el = cmd.find(qn("Order"))
        path_el = cmd.find(qn("Path"))
        desc_el = cmd.find(qn("Description"))
        data["specialize_commands"].append(dict({
            "order": order_el.text.strip() if order_el is not None and order_el.text else "",
            "path": path_el.text.strip() if path_el is not None and path_el.text else "",
            "description": desc_el.text.strip() if desc_el is not None and desc_el.text else "",
        }, **_notes(cmd)))

    # --- oobeSystem ---
    for comp in root.xpath("u:settings[@pass='oobeSystem']/u:component", namespaces=ns):
        comp_name = comp.get("name", "")

        if "International-Core" in comp_name:
            for tag, key in [
                ("InputLocale", "input_locale"),
                ("SystemLocale", "system_locale"),
                ("UILanguage", "ui_language"),
                ("UserLocale", "user_locale"),
            ]:
                el = comp.find(qn(tag))
                if el is not None and el.text:
                    data["intl"][key] = el.text.strip()

        if "OOBE" in comp_name or "Shell-Setup" in comp_name:
            oobe_el = comp.find(qn("OOBE"))
            if oobe_el is not None:
                for child in oobe_el:
                    local = etree.QName(child).localname
                    if local in data["oobe"] and child.text:
                        data["oobe"][local] = child.text.strip()

            ua = comp.find(qn("UserAccounts"))
            if ua is not None:
                la_container = ua.find(qn("LocalAccounts"))
                if la_container is not None:
                    for acct in la_container.findall(qn("LocalAccount")):
                        name_el = acct.find(qn("Name"))
                        group_el = acct.find(qn("Group"))
                        display_el = acct.find(qn("DisplayName"))
                        pw_el = acct.find(qn("Password"))
                        pw_val = ""
                        pw_plain = "true"
                        if pw_el is not None:
                            v = pw_el.find(qn("Value"))
                            p = pw_el.find(qn("PlainText"))
                            if v is not None and v.text:
                                pw_val = v.text.strip()
                            if p is not None and p.text:
                                pw_plain = p.text.strip()
                        data["user_accounts"].append({
                            "name": name_el.text.strip() if name_el is not None and name_el.text else "",
                            "password": pw_val,
                            "plain_text": pw_plain,
                            "group": group_el.text.strip() if group_el is not None and group_el.text else "Administrators",
                            "display_name": display_el.text.strip() if display_el is not None and display_el.text else "",
                        })

            al = comp.find(qn("AutoLogon"))
            if al is not None:
                enabled_el = al.find(qn("Enabled"))
                user_el = al.find(qn("Username"))
                count_el = al.find(qn("LogonCount"))
                pw_el = al.find(qn("Password"))
                pw_val = ""
                pw_plain = "true"
                if pw_el is not None:
                    v = pw_el.find(qn("Value"))
                    p = pw_el.find(qn("PlainText"))
                    if v is not None and v.text:
                        pw_val = v.text.strip()
                    if p is not None and p.text:
                        pw_plain = p.text.strip()
                data["autologon"] = {
                    "enabled": enabled_el.text.strip() if enabled_el is not None and enabled_el.text else "",
                    "username": user_el.text.strip() if user_el is not None and user_el.text else "",
                    "password": pw_val,
                    "plain_text": pw_plain,
                    "logon_count": count_el.text.strip() if count_el is not None and count_el.text else "",
                }

            flc = comp.find(qn("FirstLogonCommands"))
            if flc is not None:
                for sync in flc.findall(qn("SynchronousCommand")):
                    order_el = sync.find(qn("Order"))
                    cl_el = sync.find(qn("CommandLine"))
                    desc_el = sync.find(qn("Description"))
                    data["firstlogon_commands"].append(dict({
                        "order": order_el.text.strip() if order_el is not None and order_el.text else "",
                        "commandline": cl_el.text.strip() if cl_el is not None and cl_el.text else "",
                        "description": desc_el.text.strip() if desc_el is not None and desc_el.text else "",
                    }, **_notes(sync)))

            tz_el = comp.find(qn("TimeZone"))
            if tz_el is not None and tz_el.text:
                data["oobe_timezone"] = tz_el.text.strip()

    return data


def build_unattend_xml(form_data):
    """Build a complete unattend.xml string from form data dict."""
    root = etree.Element(qn("unattend"), nsmap=config.NSMAP)

    _settings_pass(root, "windowsPE")

    # --- offlineServicing: DriverPaths ---
    offline = _settings_pass(root, "offlineServicing")
    driver_paths = form_data.get("driver_paths", [])
    if driver_paths:
        comp = etree.SubElement(offline, qn("component"))
        comp.set("name", "Microsoft-Windows-PnpCustomizationsNonWinPE")
        comp.set("processorArchitecture", "amd64")
        comp.set("publicKeyToken", "31bf3856ad364e35")
        comp.set("language", "neutral")
        comp.set("versionScope", "nonSxS")
        dp_container = etree.SubElement(comp, qn("DriverPaths"))
        for idx, dp in enumerate(driver_paths, start=1):
            if not dp.strip():
                continue
            pac = etree.SubElement(dp_container, qn("PathAndCredentials"))
            pac.set(qwcm("action"), "add")
            pac.set(qwcm("keyValue"), str(idx))
            path_el = etree.SubElement(pac, qn("Path"))
            path_el.text = dp.strip()

    # --- specialize ---
    spec = _settings_pass(root, "specialize")

    shell_comp = etree.SubElement(spec, qn("component"))
    shell_comp.set("name", "Microsoft-Windows-Shell-Setup")
    shell_comp.set("processorArchitecture", "amd64")
    shell_comp.set("publicKeyToken", "31bf3856ad364e35")
    shell_comp.set("language", "neutral")
    shell_comp.set("versionScope", "nonSxS")

    for tag, key in [
        ("ComputerName", "computer_name"),
        ("RegisteredOrganization", "registered_organization"),
        ("RegisteredOwner", "registered_owner"),
        ("TimeZone", "time_zone"),
    ]:
        val = form_data.get(key, "").strip()
        if val:
            el = etree.SubElement(shell_comp, qn(tag))
            el.text = val

    spec_cmds = form_data.get("specialize_commands", [])
    if spec_cmds:
        deploy_comp = etree.SubElement(spec, qn("component"))
        deploy_comp.set("name", "Microsoft-Windows-Deployment")
        deploy_comp.set("processorArchitecture", "amd64")
        deploy_comp.set("publicKeyToken", "31bf3856ad364e35")
        deploy_comp.set("language", "neutral")
        deploy_comp.set("versionScope", "nonSxS")
        rs = etree.SubElement(deploy_comp, qn("RunSynchronous"))
        for idx, cmd in enumerate(spec_cmds, start=1):
            if not cmd.get("path", "").strip():
                continue
            rsc = etree.SubElement(rs, qn("RunSynchronousCommand"))
            rsc.set(qwcm("action"), "add")
            order_el = etree.SubElement(rsc, qn("Order"))
            order_el.text = str(idx)
            path_el = etree.SubElement(rsc, qn("Path"))
            path_el.text = cmd["path"].strip()
            desc_el = etree.SubElement(rsc, qn("Description"))
            desc_el.text = cmd.get("description", "").strip()

    # --- oobeSystem ---
    oobe_settings = _settings_pass(root, "oobeSystem")

    intl = form_data.get("intl", {})
    if any(v.strip() for v in intl.values() if v):
        intl_comp = etree.SubElement(oobe_settings, qn("component"))
        intl_comp.set("name", "Microsoft-Windows-International-Core")
        intl_comp.set("processorArchitecture", "amd64")
        intl_comp.set("publicKeyToken", "31bf3856ad364e35")
        intl_comp.set("language", "neutral")
        intl_comp.set("versionScope", "nonSxS")
        for tag, key in [
            ("InputLocale", "input_locale"),
            ("SystemLocale", "system_locale"),
            ("UILanguage", "ui_language"),
            ("UserLocale", "user_locale"),
        ]:
            val = intl.get(key, "").strip()
            if val:
                el = etree.SubElement(intl_comp, qn(tag))
                el.text = val

    oobe_comp = etree.SubElement(oobe_settings, qn("component"))
    oobe_comp.set("name", "Microsoft-Windows-Shell-Setup")
    oobe_comp.set("processorArchitecture", "amd64")
    oobe_comp.set("publicKeyToken", "31bf3856ad364e35")
    oobe_comp.set("language", "neutral")
    oobe_comp.set("versionScope", "nonSxS")

    oobe_el = etree.SubElement(oobe_comp, qn("OOBE"))
    oobe_data = form_data.get("oobe", {})
    for key in [
        "HideEULAPage",
        "HideOEMRegistrationScreen",
        "HideOnlineAccountScreens",
        "HideWirelessSetupInOOBE",
        "HideLocalAccountScreen",
        "NetworkLocation",
        "ProtectYourPC",
        "SkipUserOOBE",
        "SkipMachineOOBE",
    ]:
        val = oobe_data.get(key, "")
        if val:
            child = etree.SubElement(oobe_el, qn(key))
            child.text = str(val)

    accounts = form_data.get("user_accounts", [])
    if accounts:
        ua = etree.SubElement(oobe_comp, qn("UserAccounts"))
        la_container = etree.SubElement(ua, qn("LocalAccounts"))
        for acct in accounts:
            if not acct.get("name", "").strip():
                continue
            la = etree.SubElement(la_container, qn("LocalAccount"))
            la.set(qwcm("action"), "add")
            pw = etree.SubElement(la, qn("Password"))
            pw_val = etree.SubElement(pw, qn("Value"))
            pw_val.text = acct.get("password", "")
            pw_plain = etree.SubElement(pw, qn("PlainText"))
            pw_plain.text = acct.get("plain_text", "true")
            name_el = etree.SubElement(la, qn("Name"))
            name_el.text = acct["name"].strip()
            group_el = etree.SubElement(la, qn("Group"))
            group_el.text = acct.get("group", "Administrators").strip()
            display_el = etree.SubElement(la, qn("DisplayName"))
            display_el.text = acct.get("display_name", acct["name"]).strip()

    autologon = form_data.get("autologon", {})
    if autologon.get("username", "").strip():
        al = etree.SubElement(oobe_comp, qn("AutoLogon"))
        al_pw = etree.SubElement(al, qn("Password"))
        al_pw_val = etree.SubElement(al_pw, qn("Value"))
        al_pw_val.text = autologon.get("password", "")
        al_pw_plain = etree.SubElement(al_pw, qn("PlainText"))
        al_pw_plain.text = autologon.get("plain_text", "true")
        al_enabled = etree.SubElement(al, qn("Enabled"))
        al_enabled.text = autologon.get("enabled", "true")
        al_user = etree.SubElement(al, qn("Username"))
        al_user.text = autologon["username"].strip()
        logon_count = autologon.get("logon_count", "").strip()
        if logon_count:
            al_count = etree.SubElement(al, qn("LogonCount"))
            al_count.text = logon_count

    fl_cmds = form_data.get("firstlogon_commands", [])
    if fl_cmds:
        flc = etree.SubElement(oobe_comp, qn("FirstLogonCommands"))
        for idx, cmd in enumerate(fl_cmds, start=1):
            if not cmd.get("commandline", "").strip():
                continue
            sc = etree.SubElement(flc, qn("SynchronousCommand"))
            sc.set(qwcm("action"), "add")
            order_el = etree.SubElement(sc, qn("Order"))
            order_el.text = str(idx)
            cl_el = etree.SubElement(sc, qn("CommandLine"))
            cl_el.text = cmd["commandline"].strip()
            desc_el = etree.SubElement(sc, qn("Description"))
            desc_el.text = cmd.get("description", "").strip()

    oobe_tz = form_data.get("oobe_timezone", "").strip()
    if oobe_tz:
        tz_el = etree.SubElement(oobe_comp, qn("TimeZone"))
        tz_el.text = oobe_tz

    xml_bytes = etree.tostring(
        root,
        pretty_print=True,
        xml_declaration=True,
        encoding="utf-8",
    )
    return xml_bytes.decode("utf-8")


def extract_form_data(form):
    """Pull structured data from the submitted form (the field names of the
    old one-page editor). A group of fields that is not in the post at all
    is left out of the result, so apply_form leaves that part of the file
    alone: a post that carries only a few fields must not empty every list
    it did not send."""
    data = {}

    dp_list = form.getlist("driver_path[]")
    data["driver_paths"] = [p for p in dp_list if p.strip()]

    data["computer_name"] = form.get("computer_name", "")
    data["registered_organization"] = form.get("registered_organization", "")
    data["registered_owner"] = form.get("registered_owner", "")
    data["time_zone"] = form.get("time_zone", "")

    spec_paths = form.getlist("spec_cmd_path[]")
    spec_descs = form.getlist("spec_cmd_desc[]")
    data["specialize_commands"] = []
    for i in range(len(spec_paths)):
        if spec_paths[i].strip():
            data["specialize_commands"].append({
                "path": spec_paths[i],
                "description": spec_descs[i] if i < len(spec_descs) else "",
            })

    data["oobe"] = {}
    for key in [
        "HideEULAPage",
        "HideOEMRegistrationScreen",
        "HideOnlineAccountScreens",
        "HideWirelessSetupInOOBE",
        "HideLocalAccountScreen",
        "SkipUserOOBE",
        "SkipMachineOOBE",
    ]:
        data["oobe"][key] = form.get(f"oobe_{key}", "false")
    data["oobe"]["NetworkLocation"] = form.get("oobe_NetworkLocation", "Work")
    data["oobe"]["ProtectYourPC"] = form.get("oobe_ProtectYourPC", "3")

    fl_cls = form.getlist("fl_cmd_commandline[]")
    fl_descs = form.getlist("fl_cmd_desc[]")
    data["firstlogon_commands"] = []
    for i in range(len(fl_cls)):
        if fl_cls[i].strip():
            data["firstlogon_commands"].append({
                "commandline": fl_cls[i],
                "description": fl_descs[i] if i < len(fl_descs) else "",
            })

    accounts = []
    i = 0
    while form.get(f"account_name_{i}"):
        accounts.append({
            "name": form.get(f"account_name_{i}", ""),
            "password": form.get(f"account_password_{i}", ""),
            "plain_text": form.get(f"account_plaintext_{i}", "true"),
            "group": form.get(f"account_group_{i}", "Administrators"),
            "display_name": form.get(f"account_display_{i}", ""),
        })
        i += 1
    data["user_accounts"] = accounts

    data["autologon"] = {
        "enabled": form.get("autologon_enabled", ""),
        "username": form.get("autologon_username", ""),
        "password": form.get("autologon_password", ""),
        "plain_text": form.get("autologon_plaintext", "true"),
        "logon_count": form.get("autologon_logoncount", ""),
    }

    data["intl"] = {
        "input_locale": form.get("intl_input_locale", ""),
        "system_locale": form.get("intl_system_locale", ""),
        "ui_language": form.get("intl_ui_language", ""),
        "user_locale": form.get("intl_user_locale", ""),
    }

    data["oobe_timezone"] = form.get("oobe_timezone", "")

    sent = set(form.keys())
    groups = {
        "driver_paths": {"driver_path[]"},
        "computer_name": {"computer_name"},
        "registered_organization": {"registered_organization"},
        "registered_owner": {"registered_owner"},
        "time_zone": {"time_zone"},
        "specialize_commands": {"spec_cmd_path[]", "spec_cmd_desc[]"},
        "oobe": {f"oobe_{k}" for k in OOBE_KEYS},
        "firstlogon_commands": {"fl_cmd_commandline[]", "fl_cmd_desc[]"},
        "user_accounts": {k for k in sent if k.startswith("account_")},
        "autologon": {k for k in sent if k.startswith("autologon_")},
        "intl": {k for k in sent if k.startswith("intl_")},
        "oobe_timezone": {"oobe_timezone"},
    }
    return {k: v for k, v in data.items() if groups[k] & sent}


# ---------------------------------------------------------------------------
# Form edits: apply a form dict to the EXISTING file text
# ---------------------------------------------------------------------------
#
# The live answer files are hand-maintained: comments explain commands,
# processorArchitecture is the "*arch*" token PESetup substitutes, attributes
# are laid out over several lines, the shopfloor file declares xmlns="" on
# PathAndCredentials. Re-serializing the document (lxml, or
# build_unattend_xml) changes all of that, so a save here splices text:
#
#   * a value that did not change is not touched (compared with what
#     parse_unattend_text reads from the current file);
#   * a changed value replaces only the text between its tags;
#   * list items (commands, driver paths, accounts) are cut out of the file
#     as whole lines and put back in the new order. The comments directly
#     above an item, and any inside it, move with it; when an item is
#     removed, the comments above it stay and go to the next item;
#   * Order (commands) and wcm:keyValue (driver paths) are renumbered as
#     little as possible: a removed item leaves a gap, an added or moved one
#     takes a free number between its neighbours, and only when there is no
#     room do the numbers after it shift (see _numbering);
#   * a new element copies its indentation from its siblings (tabs or
#     spaces, the file's line endings), a new list item copies its
#     sibling's start tag (wcm:action, xmlns="").
#
# The result is then parsed again and compared with the form; any
# difference refuses the save (ApplyError) rather than writing something
# other than what the page showed. Missing keys in the form dict mean
# "unchanged", so the JSON API can send only what it edits.

WCM_DECL = 'xmlns:wcm="%s"' % config.WCM_NS
COMMON_COMPONENT_ATTRS = ('publicKeyToken="31bf3856ad364e35" language="neutral" '
                          'versionScope="nonSxS"')
OOBE_KEYS = ("HideEULAPage", "HideOEMRegistrationScreen", "HideOnlineAccountScreens",
             "HideWirelessSetupInOOBE", "HideLocalAccountScreen", "NetworkLocation",
             "ProtectYourPC", "SkipUserOOBE", "SkipMachineOOBE")
INTL_KEYS = (("InputLocale", "input_locale"), ("SystemLocale", "system_locale"),
             ("UILanguage", "ui_language"), ("UserLocale", "user_locale"))
SHELL_KEYS = (("ComputerName", "computer_name"),
              ("RegisteredOrganization", "registered_organization"),
              ("RegisteredOwner", "registered_owner"), ("TimeZone", "time_zone"))


class ApplyError(ValueError):
    """The form cannot be applied to this file exactly; use Raw XML."""


_TOKEN = re.compile(
    r"<!--.*?-->"
    r"|<\?.*?\?>"
    r"|<!\[CDATA\[.*?\]\]>"
    r"|<!DOCTYPE[^>]*>"
    r"|</(?P<close>[^\s>]+)\s*>"
    r"|<(?P<open>[^\s/>!?]+)(?P<attrs>(?:\s+[^\s=/>]+\s*=\s*(?:\"[^\"]*\"|'[^']*'))*)"
    r"\s*(?P<selfclose>/?)>",
    re.S)


class _Node:
    """One element or comment of the file, as character offsets."""
    __slots__ = ("kind", "tag", "qname", "start", "end", "cstart", "cend",
                 "attrs", "children", "parent")

    def __init__(self, kind, tag="", qname="", start=0, attrs="", parent=None):
        self.kind, self.tag, self.qname = kind, tag, qname
        self.start, self.end = start, None
        self.cstart = self.cend = None  # content range; None when <x/>
        self.attrs, self.children, self.parent = attrs, [], parent

    def els(self, tag=None):
        return [c for c in self.children if c.kind == "el" and (tag is None or c.tag == tag)]

    def first(self, *path):
        node = self
        for tag in path:
            found = node.els(tag)
            if not found:
                return None
            node = found[0]
        return node

    def descendants(self, tag):
        out = []
        for c in self.els():
            if c.tag == tag:
                out.append(c)
            out.extend(c.descendants(tag))
        return out

    def attr(self, name):
        m = re.search(r"(?:^|\s)(?:[\w.-]+:)?%s\s*=\s*(\"([^\"]*)\"|'([^']*)')" % re.escape(name),
                      self.attrs)
        if not m:
            return None
        return html.unescape(m.group(2) if m.group(2) is not None else m.group(3))


def _scan(text):
    """Element/comment tree of a well-formed document, as offsets."""
    doc = _Node("doc", start=0)
    stack = [doc]
    for m in _TOKEN.finditer(text):
        tok = m.group(0)
        if m.group("open"):
            q = m.group("open")
            n = _Node("el", q.split(":")[-1], q, m.start(), m.group("attrs") or "", stack[-1])
            stack[-1].children.append(n)
            if m.group("selfclose"):
                n.end = m.end()
            else:
                n.cstart = m.end()
                stack.append(n)
        elif m.group("close"):
            n = stack.pop()
            if n.kind != "el" or n.qname != m.group("close"):
                raise ApplyError("could not follow the element structure of the file")
            n.cend, n.end = m.start(), m.end()
        elif tok.startswith("<!--"):
            c = _Node("comment", start=m.start(), parent=stack[-1])
            c.end = m.end()
            stack[-1].children.append(c)
    if len(stack) != 1 or not doc.els():
        raise ApplyError("could not follow the element structure of the file")
    return doc.els()[0]


def _esc(value):
    return value.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")


def _line_start(text, pos):
    return text.rfind("\n", 0, pos) + 1


def _indent_of(text, node):
    """Whitespace before node on its line ("" when something else is there)."""
    ls = _line_start(text, node.start)
    pre = text[ls:node.start]
    return pre if not pre.strip() else ""


def _line_span(text, start, end):
    """[start, end) widened to whole lines when nothing else is on them."""
    ls = _line_start(text, start)
    nl = text.find("\n", end)
    eol = len(text) if nl < 0 else nl + 1
    if text[ls:start].strip() or text[end:eol].strip():
        return start, end
    return ls, eol


class _Edits:
    """Splices collected against the ORIGINAL text, applied in one pass."""

    def __init__(self):
        self.items = []  # (start, end, seq, text)

    def add(self, start, end, repl):
        self.items.append((start, end, len(self.items), repl))

    def render(self, text, a=0, b=None):
        b = len(text) if b is None else b
        out, pos = [], a
        for start, end, _seq, repl in sorted(self.items):
            if start < a or end > b:
                continue
            if start < pos:
                raise ApplyError("two edits overlap in the same place of the file")
            out.append(text[pos:start])
            out.append(repl)
            pos = end
        out.append(text[pos:b])
        return "".join(out)


class _Doc:
    """The file being edited: text, tree, layout and pending edits."""

    def __init__(self, text):
        self.bom = text.startswith("\ufeff")
        self.text = text[1:] if self.bom else text
        self.nl = "\r\n" if "\r\n" in self.text else "\n"
        self.root = _scan(self.text)
        first = self.root.els()
        ind = _indent_of(self.text, first[0]) if first else ""
        self.unit = ind if ind and ind.strip(" \t") == "" else "  "
        self.edits = _Edits()

    # --- layout ----------------------------------------------------------
    def indent(self, node):
        return _indent_of(self.text, node)

    def child_indent(self, parent):
        kids = parent.els()
        if kids and self.indent(kids[0]):
            return self.indent(kids[0])
        return self.indent(parent) + self.unit

    def block(self, spec, indent):
        """Lines for ("Tag", attrs, value-or-children) at indent."""
        tag, attrs, body = spec
        open_tag = "<%s%s>" % (tag, (" " + attrs) if attrs else "")
        if not isinstance(body, list):
            return "%s%s%s</%s>" % (indent, open_tag, _esc(body), tag)
        inner = [self.block(c, indent + self.unit) for c in body]
        return self.nl.join([indent + open_tag] + inner + [indent + "</%s>" % tag])

    def wcm_in_scope(self, node):
        while node is not None and node.kind == "el":
            if re.search(r"xmlns:wcm\s*=", node.attrs):
                return True
            node = node.parent
        return False

    # --- edits -----------------------------------------------------------
    def set_value(self, edits, node, value):
        """Replace the text inside node (an element with no children)."""
        if node.els():
            raise ApplyError("<%s> holds elements, not a value" % node.tag)
        if node.cstart is None:  # <x/> becomes <x>value</x>
            start_tag = re.sub(r"\s*/>$", ">", self.text[node.start:node.end])
            edits.add(node.start, node.end, "%s%s</%s>" % (start_tag, _esc(value), node.qname))
        else:
            edits.add(node.cstart, node.cend, _esc(value))

    def remove(self, edits, node):
        a, b = _line_span(self.text, node.start, node.end)
        edits.add(a, b, "")

    def insert(self, edits, parent, spec, first=False):
        """Add an element as the last (or first) child of parent."""
        t, nl = self.text, self.nl
        ind = self.child_indent(parent)
        lines = self.block(spec, ind)
        if parent.cstart is None:  # <parent/>: open it up
            start_tag = re.sub(r"\s*/>$", ">", t[parent.start:parent.end])
            edits.add(parent.start, parent.end, "%s%s%s%s%s</%s>" % (
                start_tag, nl, lines, nl, self.indent(parent), parent.qname))
            return
        kids = parent.children
        if first and kids:
            edits.add(parent.cstart, parent.cstart, nl + lines)
            return
        ls = _line_start(t, parent.cend)
        if ls > parent.cstart and not t[ls:parent.cend].strip():
            edits.add(ls, ls, lines + nl)  # closing tag on its own line
        else:
            edits.add(parent.cend, parent.cend, nl + lines + nl + self.indent(parent))

    def result(self):
        out = self.edits.render(self.text)
        return ("\ufeff" + out) if self.bom else out


# --- what the form asked for --------------------------------------------------

def _one_line(value):
    """Commands are one line in the file; a pasted line break is a space."""
    return re.sub(r"[\r\n]+", " ", str(value or "")).strip()


def _orig(row):
    try:
        n = row.get("orig")
        return int(n) if n is not None and str(n) != "" else None
    except (TypeError, ValueError, AttributeError):
        return None


def normalize_form(form):
    """The form dict cleaned the way it will be written: values stripped,
    empty list rows dropped, a blank account group is Administrators.
    Keys that are not in form are not in the result (unchanged)."""
    out = {}
    for _tag, key in SHELL_KEYS:
        if key in form:
            out[key] = str(form.get(key) or "").strip()
    if "oobe_timezone" in form:
        out["oobe_timezone"] = str(form.get("oobe_timezone") or "").strip()
    if "driver_paths" in form:
        rows = []
        for r in form.get("driver_paths") or []:
            r = r if isinstance(r, dict) else {"path": r}
            path = _one_line(r.get("path"))
            if path:
                rows.append({"path": path, "orig": _orig(r)})
        out["driver_paths"] = rows
    for key, field in (("specialize_commands", "path"), ("firstlogon_commands", "commandline")):
        if key in form:
            rows = []
            for r in form.get(key) or []:
                if not isinstance(r, dict):
                    continue
                cmd = _one_line(r.get(field))
                if cmd:
                    rows.append({field: cmd, "description": _one_line(r.get("description")),
                                 "orig": _orig(r)})
            out[key] = rows
    if "oobe" in form:
        oobe = form.get("oobe") or {}
        out["oobe"] = {k: str(oobe.get(k) or "").strip() for k in OOBE_KEYS if k in oobe}
    if "intl" in form:
        intl = form.get("intl") or {}
        out["intl"] = {key: str(intl.get(key) or "").strip() for _t, key in INTL_KEYS if key in intl}
    if "user_accounts" in form:
        rows = []
        for r in form.get("user_accounts") or []:
            if not isinstance(r, dict) or not str(r.get("name") or "").strip():
                continue
            rows.append({
                "name": str(r.get("name")).strip(),
                "password": str(r.get("password") or ""),
                "plain_text": str(r.get("plain_text") or "true").strip() or "true",
                "group": str(r.get("group") or "").strip() or "Administrators",
                "display_name": str(r.get("display_name") or "").strip(),
                "orig": _orig(r),
            })
        out["user_accounts"] = rows
    if "autologon" in form:
        al = form.get("autologon") or {}
        out["autologon"] = {
            "enabled": str(al.get("enabled") or "").strip(),
            "username": str(al.get("username") or "").strip(),
            "password": str(al.get("password") or ""),
            "plain_text": str(al.get("plain_text") or "true").strip() or "true",
            "logon_count": str(al.get("logon_count") or "").strip(),
        }
    return out


# --- locating things in the scanned file ----------------------------------------

def _settings(doc, pass_name):
    for s in doc.root.els("settings"):
        if s.attr("pass") == pass_name:
            return s
    return None


def _components(settings, contains):
    if settings is None:
        return []
    return [c for c in settings.els("component") if any(x in (c.attr("name") or "") for x in contains)]


def _arch(doc):
    for c in doc.root.descendants("component"):
        if c.attr("processorArchitecture"):
            return c.attr("processorArchitecture")
    return "amd64"


def _component_spec(doc, name, children):
    attrs = 'name="%s" processorArchitecture="%s" %s' % (name, _arch(doc), COMMON_COMPONENT_ATTRS)
    if not re.search(r"xmlns:wcm\s*=", doc.root.attrs):
        attrs += " " + WCM_DECL
    return ("component", attrs, children)


# Structures a value needs, outermost first: (check, create) pairs. Each
# create() adds ONE element; the caller re-scans and checks again, so a
# component and the list inside it are made in two rounds.

def _need_structure(doc, want, old):
    """The next missing element the form needs, as a function adding it to
    doc.edits, or None when everything needed exists."""
    spec_s, oobe_s = _settings(doc, "specialize"), _settings(doc, "oobeSystem")
    off_s = _settings(doc, "offlineServicing")

    def changed(key):
        return key in want and want[key] != old.get(key)

    needs_shell = any(changed(k) and want[k] for _t, k in SHELL_KEYS)
    needs_dp = bool(want.get("driver_paths"))
    needs_rs = bool(want.get("specialize_commands"))
    needs_intl = any(v and v != old["intl"].get(k) for k, v in (want.get("intl") or {}).items())
    needs_oobe = any(v and v != old["oobe"].get(k) for k, v in (want.get("oobe") or {}).items())
    needs_acct = bool(want.get("user_accounts"))
    needs_al = bool((want.get("autologon") or {}).get("username"))
    needs_fl = bool(want.get("firstlogon_commands"))
    needs_tz = changed("oobe_timezone") and want["oobe_timezone"]
    needs_oobe_comp = needs_oobe or needs_acct or needs_al or needs_fl or needs_tz

    for pass_name, needed, node in (("offlineServicing", needs_dp, off_s),
                                    ("specialize", needs_shell or needs_rs, spec_s),
                                    ("oobeSystem", needs_intl or needs_oobe_comp, oobe_s)):
        if needed and node is None:
            return lambda: doc.insert(doc.edits, doc.root,
                                      ("settings", 'pass="%s"' % pass_name, []))

    def comp(settings, contains, name):
        found = _components(settings, contains)
        if found:
            return found[0], None
        return None, lambda: doc.insert(doc.edits, settings, _component_spec(doc, name, []))

    if needs_dp:
        found = [p.parent for p in off_s.descendants("PathAndCredentials")]
        if not found:
            c, make = comp(off_s, ("PnpCustomizationsNonWinPE",),
                           "Microsoft-Windows-PnpCustomizationsNonWinPE")
            if make:
                return make
            if c.first("DriverPaths") is None:
                return lambda: doc.insert(doc.edits, c, ("DriverPaths", "", []))
    if needs_shell:
        c, make = comp(spec_s, ("Shell-Setup",), "Microsoft-Windows-Shell-Setup")
        if make:
            return make
    if needs_rs and not spec_s.descendants("RunSynchronousCommand"):
        c, make = comp(spec_s, ("Microsoft-Windows-Deployment",), "Microsoft-Windows-Deployment")
        if make:
            return make
        if c.first("RunSynchronous") is None:
            return lambda: doc.insert(doc.edits, c, ("RunSynchronous", "", []))
    if needs_intl:
        c, make = comp(oobe_s, ("International-Core",), "Microsoft-Windows-International-Core")
        if make:
            return make
    if needs_oobe_comp:
        c, make = comp(oobe_s, ("Shell-Setup",), "Microsoft-Windows-Shell-Setup")
        if make:
            return make
        if needs_oobe and c.first("OOBE") is None:
            return lambda: doc.insert(doc.edits, c, ("OOBE", "", []))
        if needs_acct and c.first("UserAccounts", "LocalAccounts") is None:
            ua = c.first("UserAccounts")
            if ua is None:
                return lambda: doc.insert(doc.edits, c, ("UserAccounts", "", [("LocalAccounts", "", [])]))
            return lambda: doc.insert(doc.edits, ua, ("LocalAccounts", "", []))
        if needs_al and c.first("AutoLogon") is None:
            return lambda: doc.insert(doc.edits, c, ("AutoLogon", "", [("Password", "", [])]))
        if needs_fl and c.first("FirstLogonCommands") is None:
            return lambda: doc.insert(doc.edits, c, ("FirstLogonCommands", "", []))
    return None


def _add_structure(text, want, old):
    """Create every element the form needs that the file lacks (a component,
    a list container, ...), one at a time, re-scanning after each."""
    for _round in range(20):
        doc = _Doc(text)
        make = _need_structure(doc, want, old)
        if make is None:
            return text
        make()
        text = doc.result()
    raise ApplyError("could not add the missing sections to the file")


# --- lists -------------------------------------------------------------------------

def _numbering(olds):
    """New Order/keyValue numbers for rows in their new order. olds holds
    each row's current number (None for a new row). As many rows as
    possible keep their number, so a move or an insert changes a few
    lines rather than every line below it; the rest get the free numbers
    in between, counting up from the last kept one.

    Row j can keep v_j if the rows before it fit below it: with w_j =
    v_j - j (0-based), a kept set is valid when its w values never fall
    and the first is at least 1 (numbers start at 1). The largest such
    set is a longest non-decreasing subsequence of w."""
    n = len(olds)
    w = [o - j if isinstance(o, int) and o > 0 else None for j, o in enumerate(olds)]
    best, prev = [0] * n, [-1] * n
    for j in range(n):
        if w[j] is None or w[j] < 1:
            continue
        best[j] = 1
        for i in range(j):
            if best[i] and w[i] <= w[j] and best[i] + 1 > best[j]:
                best[j], prev[j] = best[i] + 1, i
    keep = set()
    j = max(range(n), key=lambda k: best[k]) if n else -1
    while j >= 0 and best[j]:
        keep.add(j)
        j = prev[j]
    out, last = [], 0
    for j in range(n):
        last = olds[j] if j in keep else last + 1
        out.append(last)
    return out


def _int_or_none(value):
    try:
        return int(str(value).strip())
    except (TypeError, ValueError):
        return None


def _apply_list(doc, label, items, rows, match_field, old_rows, render_item, new_spec):
    """Reassemble one list container. items: the list's elements in the
    file; rows: the wanted rows (dicts with "orig"); old_rows: parsed rows
    of the current file (same order as items). render_item(edits, node,
    row, old_row, number) adds the in-item edits; new_spec(row, number)
    gives the children of a new item."""
    t = doc.text
    if len(items) != len(old_rows):
        raise ApplyError("%s: the file holds items the form cannot read; use Raw XML" % label)
    parents = {id(n.parent) for n in items}
    if len(parents) > 1:
        raise ApplyError("%s: items sit in more than one list in the file; use Raw XML" % label)

    # Which current item each row reuses: its "orig" index, else the first
    # unused item with the same command/path/name, else none (new item).
    # Every explicit orig is claimed first, so a new row that repeats an
    # existing command does not take that command's comments and Order.
    used, picks = set(), [None] * len(rows)
    for j, r in enumerate(rows):
        k = r.get("orig")
        if k is not None and 0 <= k < len(items) and k not in used:
            used.add(k)
            picks[j] = k
    for j, r in enumerate(rows):
        if picks[j] is None:
            k = next((i for i, o in enumerate(old_rows)
                      if i not in used and o.get(match_field) == r[match_field]), None)
            if k is not None:
                used.add(k)
                picks[j] = k

    if not items and not rows:
        return
    olds = [_int_or_none(old_rows[k].get("_number")) if k is not None else None for k in picks]
    if picks == list(range(len(items))) and None not in olds:
        # Same items in the same order (values may be edited): the numbers
        # stay as they are, even when the file's Orders are not in file
        # order. Windows runs commands by Order, so renumbering an untouched
        # list would change what runs when.
        numbers = olds
    else:
        numbers = _numbering(olds)

    if not rows:  # emptied: the container goes
        container = items[0].parent
        target = container
        if container.tag == "LocalAccounts" and len(container.parent.els()) == 1:
            target = container.parent
        doc.remove(doc.edits, target)
        return

    container = items[0].parent
    # Cut every item into gap (blank lines before it), lead (comments above
    # it) and body (its own lines). Items must sit on lines of their own.
    segs = []
    prev_end = container.cstart
    for n in items:
        sibs = n.parent.children
        i = sibs.index(n)
        first = n
        while i > 0 and sibs[i - 1].kind == "comment":
            i -= 1
            first = sibs[i]
        own_a, body_a = _line_start(t, first.start), _line_start(t, n.start)
        span = _line_span(t, n.start, n.end)
        if (t[own_a:first.start].strip() or t[body_a:n.start].strip()
                or span[0] != body_a):
            raise ApplyError("%s: an item shares a line with other XML; use Raw XML" % label)
        own_b = span[1]
        if own_a < prev_end:
            raise ApplyError("%s: could not separate the items; use Raw XML" % label)
        segs.append({"gap": t[prev_end:own_a], "lead": t[own_a:body_a], "a": body_a, "b": own_b})
        prev_end = own_b
    prefix = segs[0]["gap"]  # up to the first item: stays where it is
    segs[0]["gap"] = ""
    tail = t[prev_end:container.cend]

    # Comments above a removed item stay: they go to the next kept item
    # (in file order), or before the closing tag when none is left.
    carry, pending = {}, ""
    for k, s in enumerate(segs):
        if k in used:
            if pending:
                carry[k] = pending
                pending = ""
        else:
            pending += s["gap"] + s["lead"]
    tail = pending + tail

    ind = doc.indent(items[0])
    start_tpl = t[items[0].start:(items[0].cstart or items[0].end)]
    start_tpl = re.sub(r"\s*/>$", ">", start_tpl) if items[0].cstart is None else start_tpl

    out = [prefix]
    for r, k, num in zip(rows, picks, numbers):
        if k is not None:
            s = segs[k]
            edits = _Edits()
            render_item(edits, items[k], r, old_rows[k], num)
            out.append(carry.get(k, "") + s["gap"] + s["lead"] + edits.render(t, s["a"], s["b"]))
        else:
            out.append(_new_item(doc, start_tpl, ind, new_spec(r, num), num) + doc.nl)
    out.append(tail)
    doc.edits.add(container.cstart, container.cend, "".join(out))


def _new_item(doc, start_tpl, indent, children, number):
    """A new list item: its sibling's start tag (wcm:keyValue renumbered),
    children in the file's indentation."""
    start = re.sub(r"((?:[\w.-]+:)?keyValue\s*=\s*\")[^\"]*(\")",
                   lambda m: m.group(1) + str(number) + m.group(2), start_tpl)
    tag = re.match(r"<([^\s>/]+)", start).group(1)
    inner = [doc.block(c, indent + doc.unit) for c in children]
    return doc.nl.join([indent + start] + inner + [indent + "</%s>" % tag])


def _set_child(doc, edits, node, tag, new, old, first=False, keep_empty=True):
    """Make child <tag> of node hold new (old = value parsed from the file)."""
    if new == old:
        return
    el = node.first(tag)
    if el is not None:
        if new == "" and not keep_empty:
            doc.remove(edits, el)
        else:
            doc.set_value(edits, el, new)
    elif new != "":
        doc.insert(edits, node, (tag, "", new), first=first)


def _fresh_list(doc, container, label, rows, new_spec, start_tag):
    """Items for a list container that has none yet."""
    ind = doc.child_indent(container)
    if not doc.wcm_in_scope(container):
        start_tag = start_tag.replace(">", " %s>" % WCM_DECL, 1)
    numbers = list(range(1, len(rows) + 1))
    t = doc.text
    ls = _line_start(t, container.cend)
    body = "".join(_new_item(doc, start_tag.replace("{n}", str(n)), ind, new_spec(r, n), n) + doc.nl
                   for r, n in zip(rows, numbers))
    if container.cstart is None:
        open_tag = re.sub(r"\s*/>$", ">", t[container.start:container.end])
        doc.edits.add(container.start, container.end, "%s%s%s%s</%s>" % (
            open_tag, doc.nl, body, doc.indent(container), container.qname))
    elif ls > container.cstart and not t[ls:container.cend].strip():
        doc.edits.add(ls, ls, body)
    else:
        doc.edits.add(container.cend, container.cend, doc.nl + body + doc.indent(container))


def _list(doc, label, items, container, rows, field, old_rows, render_item, new_spec, start_tag):
    if not items:
        if rows:
            if container is None:
                raise ApplyError("%s: no list in the file to add to" % label)
            _fresh_list(doc, container, label, rows, new_spec, start_tag)
        return
    _apply_list(doc, label, items, rows, field, old_rows, render_item, new_spec)


# --- the whole form ------------------------------------------------------------------

def apply_form(text, form):
    """New file text: text with the form's values written into it. Raises
    ApplyError when that cannot be done exactly (the page then points the
    user at Raw XML)."""
    if text.strip() == "":
        text = UNATTEND_TEMPLATE
    try:
        etree.fromstring(text.lstrip("\ufeff").encode("utf-8"))
    except etree.XMLSyntaxError as exc:
        raise ApplyError("the current file is not valid XML (%s); fix it in Raw XML" % exc)
    old = parse_unattend_text(text)
    want = normalize_form(form)
    text = _add_structure(text, want, old)
    old = parse_unattend_text(text)  # same values; new structure only
    doc = _Doc(text)
    t = doc.text
    E = doc.edits

    off_s, spec_s, oobe_s = (_settings(doc, p) for p in ("offlineServicing", "specialize", "oobeSystem"))

    # specialize Shell-Setup: name, org, owner, time zone
    shell = (_components(spec_s, ("Shell-Setup",)) or [None])[0]
    for tag, key in SHELL_KEYS:
        if key in want and shell is not None:
            _set_child(doc, E, shell, tag, want[key], old[key], keep_empty=False)

    # driver paths (PathAndCredentials, any namespace)
    if "driver_paths" in want:
        items = off_s.descendants("PathAndCredentials") if off_s is not None else []
        container = items[0].parent if items else (off_s.descendants("DriverPaths") or [None])[0] if off_s is not None else None
        if len(old["driver_paths"]) != len(items):
            raise ApplyError("Driver paths: the file holds items the form cannot read; use Raw XML")
        old_rows = [{"path": p, "_number": n.attr("keyValue")} for p, n in zip(old["driver_paths"], items)]

        def dp_item(edits, node, row, old_row, num):
            if str(num) != str(old_row.get("_number")):
                tag_text = t[node.start:(node.cstart or node.end)]
                m = re.search(r"((?:[\w.-]+:)?keyValue\s*=\s*[\"'])([^\"']*)", tag_text)
                if m:
                    a = node.start + m.start(2)
                    edits.add(a, a + len(m.group(2)), str(num))
                else:
                    raise ApplyError("driver paths: an item has no wcm:keyValue to renumber")
            _set_child(doc, edits, node, "Path", row["path"], old_row["path"])

        _list(doc, "Driver paths", items, container, want["driver_paths"], "path", old_rows, dp_item,
              lambda r, n: [("Path", "", r["path"])],
              '<PathAndCredentials wcm:action="add" wcm:keyValue="{n}">')

    # RunSynchronous (specialize) and FirstLogonCommands (oobeSystem)
    def cmd_item(field, tagname):
        def render(edits, node, row, old_row, num):
            _set_child(doc, edits, node, "Order", str(num), old_row["order"], first=True)
            _set_child(doc, edits, node, tagname, row[field], old_row[field])
            _set_child(doc, edits, node, "Description", row["description"], old_row["description"])
        return render

    def cmd_spec(field, tagname):
        def spec(row, num):
            kids = [("Order", "", str(num)), (tagname, "", row[field])]
            if row["description"]:
                kids.append(("Description", "", row["description"]))
            return kids
        return spec

    if "specialize_commands" in want:
        items = spec_s.descendants("RunSynchronousCommand") if spec_s is not None else []
        container = (spec_s.descendants("RunSynchronous") or [None])[0] if spec_s is not None else None
        old_rows = [dict(r, _number=r["order"]) for r in old["specialize_commands"]]
        _list(doc, "Specialize commands", items, container, want["specialize_commands"], "path",
              old_rows, cmd_item("path", "Path"), cmd_spec("path", "Path"),
              '<RunSynchronousCommand wcm:action="add">')

    oobe_comps = _components(oobe_s, ("OOBE", "Shell-Setup"))
    oobe_comp = (_components(oobe_s, ("Shell-Setup",)) or [None])[0]

    if "firstlogon_commands" in want:
        items = [n for c in oobe_comps for n in c.descendants("SynchronousCommand")
                 if n.parent.tag == "FirstLogonCommands"]
        container = oobe_comp.first("FirstLogonCommands") if oobe_comp is not None else None
        old_rows = [dict(r, _number=r["order"]) for r in old["firstlogon_commands"]]
        _list(doc, "First logon commands", items, container, want["firstlogon_commands"],
              "commandline", old_rows, cmd_item("commandline", "CommandLine"),
              cmd_spec("commandline", "CommandLine"), '<SynchronousCommand wcm:action="add">')

    if "user_accounts" in want:
        items = [n for c in oobe_comps for n in c.descendants("LocalAccount")]
        container = oobe_comp.first("UserAccounts", "LocalAccounts") if oobe_comp is not None else None

        def acct_item(edits, node, row, old_row, _num):
            pw = node.first("Password")
            if pw is None and (row["password"] != old_row["password"]
                               or row["plain_text"] != old_row["plain_text"]):
                doc.insert(edits, node, ("Password", "", [("Value", "", row["password"]),
                                                          ("PlainText", "", row["plain_text"])]),
                           first=True)
            elif pw is not None:
                _set_child(doc, edits, pw, "Value", row["password"], old_row["password"])
                _set_child(doc, edits, pw, "PlainText", row["plain_text"], old_row["plain_text"])
            _set_child(doc, edits, node, "Name", row["name"], old_row["name"])
            _set_child(doc, edits, node, "Group", row["group"], old_row["group"])
            _set_child(doc, edits, node, "DisplayName", row["display_name"], old_row["display_name"])

        def acct_spec(row, _num):
            kids = [("Password", "", [("Value", "", row["password"]), ("PlainText", "", row["plain_text"])]),
                    ("Name", "", row["name"]), ("Group", "", row["group"])]
            if row["display_name"]:
                kids.append(("DisplayName", "", row["display_name"]))
            return kids

        _list(doc, "User accounts", items, container, want["user_accounts"], "name",
              old["user_accounts"], acct_item, acct_spec, '<LocalAccount wcm:action="add">')

    # International-Core
    if "intl" in want:
        intl = (_components(oobe_s, ("International-Core",)) or [None])[0]
        for tag, key in INTL_KEYS:
            if key in want["intl"] and intl is not None:
                _set_child(doc, E, intl, tag, want["intl"][key], old["intl"][key], keep_empty=False)

    if oobe_comp is not None:
        if "oobe" in want:
            oobe_el = oobe_comp.first("OOBE")
            for key, value in want["oobe"].items():
                if oobe_el is not None:
                    _set_child(doc, E, oobe_el, key, value, old["oobe"][key], keep_empty=False)
        if "autologon" in want:
            al_want, al_old = want["autologon"], old["autologon"]
            al = oobe_comp.first("AutoLogon")
            if al is not None and not al_want["username"]:
                doc.remove(E, al)  # no user, no AutoLogon (as the old builder did)
            elif al is not None:
                if not al_old["username"]:
                    # New block (or one with no user): parse defaults are not
                    # in the file, so write every field; Enabled defaults on.
                    al_old = {k: "" for k in al_old}
                    al_want["enabled"] = al_want["enabled"] or "true"
                pw = al.first("Password")
                if pw is None:
                    raise ApplyError("AutoLogon has no Password element; use Raw XML")
                _set_child(doc, E, pw, "Value", al_want["password"], al_old["password"])
                _set_child(doc, E, pw, "PlainText", al_want["plain_text"], al_old["plain_text"])
                _set_child(doc, E, al, "Enabled", al_want["enabled"], al_old["enabled"], keep_empty=False)
                _set_child(doc, E, al, "Username", al_want["username"], al_old["username"])
                _set_child(doc, E, al, "LogonCount", al_want["logon_count"], al_old["logon_count"],
                           keep_empty=False)
        if "oobe_timezone" in want:
            _set_child(doc, E, oobe_comp, "TimeZone", want["oobe_timezone"], old["oobe_timezone"],
                       keep_empty=False)

    new_text = doc.result()
    _check(new_text, want, old)
    return new_text


def _check(new_text, want, old):
    """Parse the result and compare it with the form: refuse on any gap.
    Orders must rise in file order, unless the list kept the Orders the
    file already had (an untouched list is never renumbered)."""
    try:
        etree.fromstring(new_text.lstrip("\ufeff").encode("utf-8"))
    except etree.XMLSyntaxError as exc:
        raise ApplyError("internal check: the result is not valid XML (%s)" % exc)
    got = parse_unattend_text(new_text)
    bad = []
    for key in [k for _t, k in SHELL_KEYS] + ["oobe_timezone"]:
        if key in want and got[key] != want[key]:
            bad.append(key)
    if "driver_paths" in want and got["driver_paths"] != [r["path"] for r in want["driver_paths"]]:
        bad.append("driver_paths")
    for key, field in (("specialize_commands", "path"), ("firstlogon_commands", "commandline")):
        if key in want:
            g = [(r[field], r["description"]) for r in got[key]]
            w = [(r[field], r["description"]) for r in want[key]]
            orders = [_int_or_none(r["order"]) for r in got[key]]
            kept = orders == [_int_or_none(r["order"]) for r in old[key]]
            if g != w or None in orders or (orders != sorted(set(orders)) and not kept):
                bad.append(key)
    if "user_accounts" in want:
        fields = ("name", "password", "plain_text", "group", "display_name")
        if [[r[f] for f in fields] for r in got["user_accounts"]] != \
                [[r[f] for f in fields] for r in want["user_accounts"]]:
            bad.append("user_accounts")
    for key in ("oobe", "intl"):
        for k, v in (want.get(key) or {}).items():
            if got[key].get(k) != v:
                bad.append("%s.%s" % (key, k))
    if "autologon" in want:
        al = want["autologon"]
        if al["username"]:
            if any(got["autologon"][k] != al[k] for k in al if not (k == "enabled" and al[k] == "")):
                bad.append("autologon")
        elif got["autologon"]["username"]:
            bad.append("autologon")
    if bad:
        raise ApplyError("internal check: the result does not hold what the form says for "
                         + ", ".join(bad) + ". Nothing was written; use Raw XML for this change.")


# ---------------------------------------------------------------------------
# Lint: the checks of scripts/lint-unattend.py, for the review step
# ---------------------------------------------------------------------------
# Keep in step with scripts/lint-unattend.py (same limits, same messages
# minus the file name). One value over a schema limit invalidates the WHOLE
# answer file for its pass, so a save that ADDS an ERROR is refused.

MAX_PATH = 259          # Microsoft-Windows-Deployment RunSynchronousCommand/Path
MAX_COMMANDLINE = 1024  # Microsoft-Windows-Shell-Setup SynchronousCommand/CommandLine
MAX_DESCRIPTION = 256   # Description on either of the above
LINT_KNOWN_TOKENS = {"%serialnumber%"}
LINT_SHELL_VARS = {"%windir%", "%systemroot%", "%systemdrive%", "%programfiles%",
                   "%programdata%", "%temp%", "%tmp%", "%userprofile%", "%appdata%",
                   "%localappdata%", "%computername%", "%username%", "%public%",
                   "%allusersprofile%", "%path%"}
LINT_SINGLE = ("Order", "Description", "Path", "CommandLine", "RequiresUserInput", "WillReboot")


def _blocks(raw, tag):
    return re.findall(r"<%s\b.*?</%s>" % (tag, tag), raw, re.S)


def _order_of(block):
    m = re.search(r"<Order>(\d+)</Order>", block)
    return m.group(1) if m else "?"


def lint_text(text):
    """Findings for an answer file's text: [{"level", "kind", "message"}]."""
    out = []

    def add(level, kind, msg):
        out.append({"level": level, "kind": kind, "message": msg})

    if text.startswith("\ufeff"):
        add("WARN", "bom", "The file carries a UTF-8 BOM. The live files have none; this is "
            "usually an accidental diff from a tool that wrote utf-8-sig.")
        raw = text[1:]
    else:
        raw = text
    try:
        etree.fromstring(raw.encode("utf-8"))
    except etree.XMLSyntaxError as exc:
        add("ERROR", "not-well-formed", "XML does not parse: %s" % exc)
        return out
    for tag, field, limit in (("RunSynchronousCommand", "Path", MAX_PATH),
                              ("SynchronousCommand", "CommandLine", MAX_COMMANDLINE)):
        for b in _blocks(raw, tag):
            m = re.search(r"<%s>(.*?)</%s>" % (field, field), b, re.S)
            if m and len(m.group(1)) > limit:
                add("ERROR", "%s-too-long" % field.lower(),
                    "%s Order %s has %s of %d chars, limit %d. This invalidates the WHOLE answer "
                    "file for its pass. Move the body into a script under Deploy\\Applications "
                    "and call it by path." % (tag, _order_of(b), field, len(m.group(1)), limit))
    for tag in ("RunSynchronousCommand", "SynchronousCommand"):
        for b in _blocks(raw, tag):
            for d in re.findall(r"<Description>(.*?)</Description>", b, re.S):
                if len(d) > MAX_DESCRIPTION:
                    add("ERROR", "description-too-long",
                        "%s Order %s has a Description of %d chars, limit %d. Put the rationale "
                        "in an XML comment instead." % (tag, _order_of(b), len(d), MAX_DESCRIPTION))
            for el in LINT_SINGLE:
                n = len(re.findall(r"<%s>" % el, b))
                if n > 1:
                    add("ERROR", "duplicate-element",
                        "%s Order %s has %d <%s> elements; at most one is allowed."
                        % (tag, _order_of(b), n, el))
    for tok in sorted(set(re.findall(r"%[a-zA-Z_]+%", raw))):
        if tok.lower() not in LINT_KNOWN_TOKENS and tok.lower() not in LINT_SHELL_VARS:
            add("WARN", "unknown-token",
                "%s is not substituted by PESetup (it replaces only %%serialnumber%% and *arch*), "
                "so it reaches Windows literally." % tok)
    return out


def lint_compare(old_text, new_text):
    """Lint of new_text, each finding marked "new" when old_text lacks it."""
    before = {(f["kind"], f["message"]) for f in lint_text(old_text)} if old_text else set()
    found = lint_text(new_text)
    for f in found:
        f["new"] = (f["kind"], f["message"]) not in before
    return {
        "findings": found,
        "errors": sum(1 for f in found if f["level"] == "ERROR"),
        "new_errors": sum(1 for f in found if f["level"] == "ERROR" and f["new"]),
    }