diff --git a/scripts/lint-driver-catalogue.py b/scripts/lint-driver-catalogue.py new file mode 100755 index 0000000..4371bc6 --- /dev/null +++ b/scripts/lint-driver-catalogue.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +""" +lint-driver-catalogue.py - Validate HardwareDriver.json against PESetup's real matcher. + +PESetup selects a driver pack with GetDriverByModel (see docs/PESETUP-INTERNALS.md) +and a miss is only a WARNING: imaging finishes, the machine comes up with no NIC and +no WiFi, DNS fails at first boot, and bulk enrollment cannot reach the CDN. The +symptom appears far from the cause, which is how the OptiPlex Micro 7020 driver pack +went unnoticed and how the Display MicroPC failures were misattributed to a drive +letter. + +This script reimplements the matcher exactly and reports the ways a catalogue can be +silently wrong: + + - a "virtual platform" entry, which fails every physical machine + - an empty token (trailing comma), which matches every model string + - tokens carrying whitespace, which the C# Split(',') never trims + - a referenced zip that is absent, or present under a different case + - an entry that can never win because an earlier entry shadows it + - a family field that contradicts the entry's own tokens + - two entries covering one model for different OS versions (first wins, not the + OS-appropriate one) + +Usage: + ./lint-driver-catalogue.py # lint every image type on the PXE server + ./lint-driver-catalogue.py --image gea-shopfloor # one image type + ./lint-driver-catalogue.py --local /srv/samba/winpeapps # run on the server itself + ./lint-driver-catalogue.py --models fleet.txt # also resolve real WMI model strings + ./lint-driver-catalogue.py --quiet # findings only, no per-entry detail + +Exits non-zero when any ERROR or CRITICAL is found, so it can gate a deploy. + +Requires: sshpass (remote mode only) +""" + +import argparse +import json +import subprocess +import sys +from pathlib import PurePosixPath + +PXE_HOST = "172.16.9.1" +PXE_USER = "pxe" +PXE_PASS = "pxe" +IMAGE_BASE = "/srv/samba/winpeapps" +# Where destinationDir points, relative to the image root. A symlink into _shared. +DRIVERS_SUBDIR = "Deploy/Out-of-box Drivers" + +# PESetup only knows these three Dell lines. Anything else skips the family filter +# and rides entirely on modelswminame substrings. +KNOWN_FAMILIES = ("Latitude", "Optiplex", "Precision") + +CRITICAL, ERROR, WARN, INFO = "CRITICAL", "ERROR", "WARN", "INFO" +FAIL_LEVELS = (CRITICAL, ERROR) + + +# --------------------------------------------------------------------------- +# Transport +# --------------------------------------------------------------------------- + +def ssh_cmd(host, cmd): + return subprocess.run( + ["sshpass", "-p", PXE_PASS, "ssh", "-o", "StrictHostKeyChecking=no", + "-o", "LogLevel=ERROR", f"{PXE_USER}@{host}", cmd], + capture_output=True, text=True) + + +def read_remote(host, path): + r = ssh_cmd(host, "cat '%s'" % path) + if r.returncode != 0: + return None + return r.stdout + + +def list_remote(host, path): + """Every file under path, relative and POSIX. Empty list if path is gone. + + -L is required: Deploy/Out-of-box Drivers is a symlink into _shared, and plain + find reports the symlink itself and descends nothing, which silently turns every + zip-existence check into a no-op.""" + r = ssh_cmd(host, "find -L '%s' -type f -printf '%%P\\n' 2>/dev/null" % path) + if r.returncode != 0: + return [] + return [line for line in r.stdout.splitlines() if line] + + +def read_local(path): + try: + with open(path) as f: + return f.read() + except OSError: + return None + + +def list_local(path): + # os.walk with followlinks, not rglob: the drivers dir is a symlink into + # _shared and rglob will not descend it. + import os + if not os.path.isdir(path): + return [] + out = [] + for root, _dirs, names in os.walk(path, followlinks=True): + rel = os.path.relpath(root, path) + for n in names: + out.append(n if rel == "." else os.path.join(rel, n)) + return out + + +# --------------------------------------------------------------------------- +# PESetup's matcher, reimplemented +# --------------------------------------------------------------------------- + +def model_family(model): + """PESetup: three Contains() tests, last one wins, "" when none hit.""" + upper = model.upper() + family = "" + if "LATITUDE" in upper: + family = "Latitude" + if "OPTIPLEX" in upper: + family = "Optiplex" + if "PRECISION" in upper: + family = "Precision" + return family + + +def entry_tokens(entry): + """modelswminame split on comma. NOT trimmed - the C# does not trim either.""" + return str(entry.get("modelswminame") or "").split(",") + + +def entry_matches(entry, model): + family = model_family(model) + if family and family.lower() not in str(entry.get("family") or "").lower(): + return False + for token in entry_tokens(entry): + # C#: model.ToLower().Contains(token.ToLower()). "" is contained by + # every string, so a trailing comma matches all models. + if token.lower() in model.lower(): + return True + return False + + +def select_driver(entries, model): + """First match wins, exactly like GetDriverByModel. None when nothing matches.""" + for i, entry in enumerate(entries): + if entry_matches(entry, model): + return i, entry + return None, None + + +# --------------------------------------------------------------------------- +# Path resolution +# --------------------------------------------------------------------------- + +def resolve_dest_dir(dest): + """*destinationdir*\\Deploy\\... -> Deploy/... (matches download-drivers.py).""" + return (str(dest or "") + .replace("*destinationdir*\\", "") + .replace("*destinationdir*", "") + .replace("\\", "/") + .strip("/")) + + +def entry_zip_relpath(entry): + dest = resolve_dest_dir(entry.get("destinationDir") or entry.get("DestinationDir")) + name = str(entry.get("fileName") or entry.get("FileName") or "") + if not name: + return None + return str(PurePosixPath(dest) / name) if dest else name + + +# --------------------------------------------------------------------------- +# Checks +# --------------------------------------------------------------------------- + +class Findings: + def __init__(self): + self.items = [] + + def add(self, level, check, message, entry_index=None): + self.items.append((level, check, message, entry_index)) + + def worst(self): + for level in (CRITICAL, ERROR, WARN, INFO): + if any(i[0] == level for i in self.items): + return level + return None + + def count(self, level): + return sum(1 for i in self.items if i[0] == level) + + +def label(entry, index): + name = entry.get("modelsfriendlyname") or entry.get("modelswminame") or "?" + return "[%02d] %s" % (index, name) + + +def check_catalogue(entries, files, findings): + """Structural checks: one pass over the entries, no share lookups.""" + # A file listing is only meaningful when the drivers tree was readable. + files_lower = {f.lower(): f for f in files} + files_set = set(files) + + for i, entry in enumerate(entries): + tag = label(entry, i) + + # 1. Virtual-platform hijack. One entry anywhere flips the whole tool + # into virtual-only mode and every physical machine hard-fails. + manufacturer = str(entry.get("manufacturer") or "") + if "virtual platform" in manufacturer.lower(): + findings.add(CRITICAL, "virtual-platform", + "%s manufacturer is %r. PESetup takes list[0] as the driver and " + "then REQUIRES the machine to look virtual, so every physical " + "machine fails GatherDataSelection." % (tag, manufacturer), i) + + # 2. Empty token. C# Contains("") is true for every string, so this entry + # swallows every model whose family passes the filter. + tokens = entry_tokens(entry) + if any(t == "" for t in tokens): + findings.add(CRITICAL, "empty-token", + "%s modelswminame %r yields an EMPTY token (trailing or doubled " + "comma). An empty token matches every model string, so this entry " + "hijacks the catalogue." % (tag, entry.get("modelswminame")), i) + + # 3. Untrimmed whitespace. Split(',') keeps the space, so the model string + # must literally contain " OptiPlex 7020" to match. + for token in tokens: + if token and token != token.strip(): + findings.add(ERROR, "token-whitespace", + "%s token %r carries whitespace. PESetup does not trim, so a " + "model must contain the space too - this token will usually " + "miss." % (tag, token), i) + + if not str(entry.get("modelswminame") or "").strip(): + findings.add(ERROR, "no-tokens", + "%s has an empty modelswminame, so no model can ever select it." + % tag, i) + + # 4. Family filter contradicting the entry's own tokens. PESetup derives the + # family from the MODEL, then requires entry.family to contain it. + family_field = str(entry.get("family") or "") + for token in tokens: + token = token.strip() + if not token: + continue + implied = model_family(token) + if implied and implied.lower() not in family_field.lower(): + findings.add(ERROR, "family-mismatch", + "%s token %r implies family %s, but family is %r. The filter " + "runs before the token test, so this token can never match." + % (tag, token, implied, family_field), i) + + # 5. Referenced zip present, case-exact. The share is a case-sensitive Linux + # filesystem; Windows-side tooling that writes "Optiplex" instead of + # "OptiPlex" splits the tree and the pack is never found. + rel = entry_zip_relpath(entry) + if not rel: + findings.add(ERROR, "no-filename", + "%s has no fileName, so CopyDrivers has nothing to unzip." % tag, i) + elif files: + if rel not in files_set: + alt = files_lower.get(rel.lower()) + if alt: + findings.add(ERROR, "case-mismatch", + "%s references %r but the share holds %r. Case-sensitive " + "filesystem: PESetup will not find it." % (tag, rel, alt), i) + else: + findings.add(ERROR, "missing-zip", + "%s references %r, which is not on the share. Driver miss " + "is a WARNING only, so imaging finishes with no drivers." + % (tag, rel), i) + + # 6. Shadowing. Replay each entry's own model names through the real matcher; + # if a different entry wins, this one is dead weight or an outright trap. + for i, entry in enumerate(entries): + for token in entry_tokens(entry): + token = token.strip() + if not token: + continue + winner_i, _ = select_driver(entries, token) + if winner_i is not None and winner_i != i: + findings.add(WARN, "shadowed", + "%s token %r is answered by %s instead (first match wins)." + % (label(entry, i), token, label(entries[winner_i], winner_i)), i) + + # 7. One model, several OS builds. GetDriverByModel does not look at osId, so + # the win10 pack can land on a win11 build purely by ordering. + by_token = {} + for i, entry in enumerate(entries): + for token in entry_tokens(entry): + token = token.strip().lower() + if token: + by_token.setdefault(token, []).append(i) + for token, idxs in sorted(by_token.items()): + if len(idxs) > 1: + detail = ", ".join("%s osId=%s" % (label(entries[i], i), entries[i].get("osId")) + for i in idxs) + findings.add(WARN, "duplicate-token", + "token %r is claimed by %d entries (%s). osId is NOT part of the " + "match, so the first one wins regardless of the OS being applied." + % (token, len(idxs), detail)) + + +def check_fleet(entries, models, findings): + """Resolve real WMI model strings. A miss here is a machine with no drivers.""" + for model in models: + idx, entry = select_driver(entries, model) + if entry is None: + findings.add(ERROR, "fleet-miss", + "model %r matches NO entry. PESetup logs 'driver for [%s] not " + "found' as a warning and images the machine with no drivers." + % (model, model)) + else: + findings.add(INFO, "fleet-match", + "model %r -> %s (%s)" % (model, label(entry, idx), + entry.get("fileName"))) + + +# --------------------------------------------------------------------------- +# Driver +# --------------------------------------------------------------------------- + +def lint_image(name, catalogue_text, files, models, quiet): + print("=" * 72) + print("IMAGE: %s" % name) + print("=" * 72) + + if catalogue_text is None: + print(" SKIP: no HardwareDriver.json") + return None + + try: + entries = json.loads(catalogue_text) + except json.JSONDecodeError as e: + print(" CRITICAL: HardwareDriver.json does not parse: %s" % e) + return CRITICAL + + if not isinstance(entries, list): + print(" CRITICAL: HardwareDriver.json is %s, expected a list" % type(entries).__name__) + return CRITICAL + + findings = Findings() + check_catalogue(entries, files, findings) + if models: + check_fleet(entries, models, findings) + + print(" %d entries, %d driver files visible on the share" % (len(entries), len(files))) + + if not quiet: + for i, entry in enumerate(entries): + print(" %-28s family=%-28s %s" % ( + label(entry, i), + (entry.get("family") or "")[:28], + entry.get("fileName") or "(no fileName)")) + + print() + order = {CRITICAL: 0, ERROR: 1, WARN: 2, INFO: 3} + for level, check, message, _ in sorted(findings.items, key=lambda f: order[f[0]]): + if quiet and level == INFO: + continue + print(" %-8s %-18s %s" % (level, check, message)) + + if not findings.items: + print(" clean") + print() + print(" %d critical, %d error, %d warn" % ( + findings.count(CRITICAL), findings.count(ERROR), findings.count(WARN))) + print() + return findings.worst() + + +def main(): + parser = argparse.ArgumentParser( + description="Lint HardwareDriver.json against PESetup's real driver matcher.") + parser.add_argument("--image", help="single image type (default: all on the server)") + parser.add_argument("--server", default=PXE_HOST, + help="PXE server IP (default: %s)" % PXE_HOST) + parser.add_argument("--local", metavar="PATH", + help="lint a local winpeapps tree instead of going over SSH") + parser.add_argument("--models", metavar="FILE", + help="file of WMI model strings, one per line, to resolve") + parser.add_argument("--quiet", action="store_true", + help="findings only, no per-entry listing") + args = parser.parse_args() + + models = [] + if args.models: + with open(args.models) as f: + models = [line.strip() for line in f + if line.strip() and not line.startswith("#")] + + if args.local: + base = args.local.rstrip("/") + reader, lister = read_local, list_local + else: + base = IMAGE_BASE + reader = lambda p: read_remote(args.server, p) + lister = lambda p: list_remote(args.server, p) + + if args.image: + images = [args.image] + elif args.local: + from pathlib import Path + images = sorted(d.name for d in Path(base).iterdir() + if d.is_dir() and not d.name.startswith("_")) + else: + r = ssh_cmd(args.server, + "find '%s' -maxdepth 1 -mindepth 1 -type d -printf '%%f\\n'" % base) + if r.returncode != 0: + sys.exit("ERROR: cannot list %s on %s: %s" % (base, args.server, r.stderr.strip())) + images = sorted(n for n in r.stdout.split() if not n.startswith("_")) + + worst = None + order = {CRITICAL: 0, ERROR: 1, WARN: 2, INFO: 3, None: 4} + for image in images: + catalogue = reader("%s/%s/Deploy/Control/HardwareDriver.json" % (base, image)) + # Drivers live under the shared tree that Deploy/Out-of-box Drivers points at. + # Listing through the image path follows the symlink, so per-image case + # differences still show up. The listing comes back relative to the drivers + # dir; destinationDir is relative to the image root, so re-anchor it. + files = ["%s/%s" % (DRIVERS_SUBDIR, f) + for f in lister("%s/%s/%s" % (base, image, DRIVERS_SUBDIR))] + result = lint_image(image, catalogue, files, models, args.quiet) + if order[result] < order[worst]: + worst = result + + if worst in FAIL_LEVELS: + print("FAILED: fix the findings above before imaging.") + return 1 + print("PASSED: no blocking findings.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())