"""Verified MCL uploads: prove every file in the upload folder is the one the Windows PC sent, repair bad ones over HTTP, and gate the import. Upload-Image.ps1 (work PC) hashes every file it copies into upload-manifest.json and POSTs that manifest here after robocopy. A background thread then hashes the files on THIS disk (no SMB client cache in the way) and records which are missing, which differ, and which were already bad in the MCL cache (their hash on the PC does not match the hash MCL's own Control JSONs publish). The script re-sends the bad small files with PUT, which writes them to a temp name, checks the hash and renames them into place. The import page only imports an upload folder whose last verification passed and whose files have not changed since (size, mtime, ctime and inode of every file), with no stray files from an earlier upload mixed in. The import holds the same lock as verification and PUT, so neither can run while files are being moved. Paths are matched against the disk case-insensitively, as Windows and Samba do: the manifest's case comes from MCL's JSON, the disk's from whatever robocopy created first. State files live at the top of the upload folder and are never imported: upload-manifest.json the manifest (written by the POST, not by SMB) .upload-verify.json progress / last result (small; polled) .upload-verify-files.json per-file hash cache and snapshot (gate only) """ import hashlib import json import os import re import secrets import threading import time from contextlib import contextmanager from datetime import datetime, timezone import config MANIFEST = "upload-manifest.json" STATE = ".upload-verify.json" DETAIL = ".upload-verify-files.json" TOP_FILES = (MANIFEST, STATE, DETAIL) PART_RE = re.compile(r"^\..+\.part-[0-9a-f]{16}$") SMALL_MAX = 64 * 1024 * 1024 # PUT limit; bigger files go back to robocopy MAX_FILES = 200_000 UPLOAD_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,80}$") SHA_RE = re.compile(r"^[0-9a-fA-F]{64}$") DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]") CHUNK = 1024 * 1024 _lock = threading.Lock() # one verify, PUT or import at a time class UploadError(ValueError): """Bad request (400) unless .status says otherwise.""" def __init__(self, msg, status=400): super().__init__(msg) self.status = status @contextmanager def exclusive(what): """Hold the upload lock or fail at once with 409.""" if not _lock.acquire(blocking=False): raise UploadError(f"busy: a verification, file upload or import is running ({what} refused)", 409) try: yield finally: _lock.release() # --- paths ------------------------------------------------------------------ def root(): return config.UPLOAD_DIR def _now(): return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") def is_reserved(rel): """Only our own files at the top of the folder (and their temp files). A '.part-' file deeper down is debris and counts as stray.""" if "/" in rel: return False if rel in TOP_FILES: return True m = PART_RE.match(rel) return bool(m) and any(rel.startswith(f".{n}.part-") for n in TOP_FILES) def norm_path(p): """Manifest path -> 'Deploy/Control/x.json'. Raises on anything that could leave the upload folder.""" if not isinstance(p, str) or not p or len(p) > 1024 or "\x00" in p: raise UploadError("bad path") p = p.replace("\\", "/") if p.startswith("/") or DRIVE_RE.match(p) or ":" in p: raise UploadError(f"absolute path not allowed: {p}") parts = p.split("/") if any(s in ("", ".", "..") for s in parts): raise UploadError(f"bad path: {p}") if is_reserved(p) or PART_RE.match(parts[-1]): raise UploadError(f"reserved name: {p}") return p def safe_join(rel, base=None): """Absolute path of rel inside base, refusing symlink escapes.""" base = os.path.realpath(base or root()) full = os.path.join(base, *rel.split("/")) real = os.path.realpath(full) if real != full or not real.startswith(base + os.sep): # realpath differs: a symlink somewhere on the way. Refuse it. raise UploadError(f"path leaves the upload folder or uses a link: {rel}") return full def scan(base=None): """Walk the upload folder once. Returns (files, links): files = {rel_lower: [rel, ...]} for regular files; links = rels of symlinks and other non-regular entries, which are never part of an upload.""" base = base or root() files, links = {}, [] for dp, dns, fns in os.walk(base): for d in list(dns): if os.path.islink(os.path.join(dp, d)): links.append(os.path.relpath(os.path.join(dp, d), base).replace(os.sep, "/")) dns.remove(d) dns.sort() for fn in sorted(fns): full = os.path.join(dp, fn) rel = os.path.relpath(full, base).replace(os.sep, "/") if is_reserved(rel): continue if os.path.islink(full) or not os.path.isfile(full): links.append(rel) continue files.setdefault(rel.lower(), []).append(rel) return files, links def _key(st): """What 'unchanged' means: robocopy keeps the source mtime, so mtime and size alone cannot tell a re-copy from the bad copy before it. ctime is set by the kernel on every write and cannot be set from userland.""" return [st.st_size, st.st_mtime_ns, st.st_ctime_ns, st.st_ino] # --- manifest --------------------------------------------------------------- def parse_manifest(data): """Validate a manifest dict; return it with normalised paths and lower-case hashes.""" if not isinstance(data, dict): raise UploadError("manifest must be a JSON object") if data.get("manifest_version") != 1: raise UploadError("unsupported manifest_version (want 1)") uid = data.get("upload_id") if not isinstance(uid, str) or not UPLOAD_ID_RE.match(uid): raise UploadError("bad upload_id") files = data.get("files") if not isinstance(files, list) or not files: raise UploadError("manifest has no files") if len(files) > MAX_FILES: raise UploadError("too many files") out, seen = [], set() for f in files: if not isinstance(f, dict): raise UploadError("bad file entry") p = norm_path(f.get("path")) size, sha = f.get("size"), f.get("sha256") if not isinstance(size, int) or isinstance(size, bool) or size < 0: raise UploadError(f"bad size for {p}") if not isinstance(sha, str) or not SHA_RE.match(sha): raise UploadError(f"bad sha256 for {p}") if p.lower() in seen: raise UploadError(f"duplicate path: {p}") seen.add(p.lower()) out.append({"path": p, "size": size, "sha256": sha.lower()}) mcl = data.get("mcl") if isinstance(data.get("mcl"), dict) else {} clean = {"manifest_version": 1, "upload_id": uid, "files": out, "mcl": mcl} for k in ("created", "source_pc"): if isinstance(data.get(k), str): clean[k] = data[k][:200] return clean def _read_json(path): with open(path, "rb") as f: return json.loads(f.read().decode("utf-8-sig")) def _write_json(path, obj, indent=1): """Atomic: temp file in the same folder, fsync, rename.""" d = os.path.dirname(path) tmp = os.path.join(d, f".{os.path.basename(path)}.part-{secrets.token_hex(8)}") with open(tmp, "w", encoding="utf-8") as f: json.dump(obj, f, indent=indent) f.flush() os.fsync(f.fileno()) os.replace(tmp, path) def load_manifest(): """(manifest, raw_sha) or (None, None) when there is none.""" p = os.path.join(root(), MANIFEST) if not os.path.isfile(p): return None, None with open(p, "rb") as f: raw = f.read() try: m = parse_manifest(json.loads(raw.decode("utf-8-sig"))) except (ValueError, UnicodeDecodeError): return None, None return m, hashlib.sha256(raw).hexdigest() def load_state(): try: return _read_json(os.path.join(root(), STATE)) except (OSError, ValueError): return None def load_detail(): try: return _read_json(os.path.join(root(), DETAIL)) except (OSError, ValueError): return {} def _chown_like_parent(path): """Match the share's owner (Samba 'force user = pxe') so a later SMB re-copy can overwrite what the webapp (root) wrote.""" try: st = os.stat(os.path.dirname(path)) os.chown(path, st.st_uid, st.st_gid) except (OSError, AttributeError): pass # --- MCL's own hashes and OS ------------------------------------------------ def _mcl_rel(dest_dir, file_name): """MCL destination ('*destinationdir*\\Deploy\\Packages\\X' or 'C:\\Deploy\\Languages\\Y') + file name -> 'Deploy/Packages/X/name'.""" d = (dest_dir or "").replace("\\", "/") d = re.sub(r"^\*destinationdir\*/?", "", d, flags=re.I) d = re.sub(r"^[A-Za-z]:/", "", d).strip("/") return f"{d}/{file_name}" if d else file_name def _control_list(base, name): try: d = _read_json(os.path.join(base, "Deploy", "Control", name)) return d if isinstance(d, list) else [] except (OSError, ValueError, UnicodeDecodeError): return [] def mcl_expected(base=None): """{path_lower: (path, sha256_lower)} for every payload file the Control JSONs in this upload describe. Only real SHA-256 values.""" base = base or root() exp = {} def add(dest, name, sha): if not (dest and name and isinstance(sha, str) and SHA_RE.match(sha)): return rel = _mcl_rel(dest, name) exp[rel.lower()] = (rel, sha.lower()) for e in _control_list(base, "OperatingSystem.json"): wim = ((e or {}).get("operatingSystemVersion") or {}).get("wim") or {} add(wim.get("destinationDir") or wim.get("DestinationDir"), wim.get("fileName") or wim.get("FileName") or "install.wim", wim.get("hash") or wim.get("Hash")) for fname in ("HardwareDriver.json", "packages.json"): for e in _control_list(base, fname): if isinstance(e, dict): add(e.get("destinationDir") or e.get("DestinationDir"), e.get("fileName") or e.get("FileName"), e.get("hash") or e.get("Hash")) for e in _control_list(base, "LanguagePacks.json"): for lf in (e or {}).get("files") or []: if isinstance(lf, dict): add(lf.get("destFolder"), lf.get("fileName"), lf.get("hash")) return exp def detected_os(manifest=None, base=None): """The OS the upload carries: the OperatingSystem.json entry whose id is the selected one (manifest mcl.os_id, else Tools/user_selections.json). A single entry is taken as is. None when it cannot be told.""" base = base or root() entries = [e.get("operatingSystemVersion") for e in _control_list(base, "OperatingSystem.json") if isinstance(e, dict) and isinstance(e.get("operatingSystemVersion"), dict)] if not entries: return None want = ((manifest or {}).get("mcl") or {}).get("os_id") if want is None: try: sel = _read_json(os.path.join(base, "Tools", "user_selections.json")) want = sel[0].get("OperatingSystemSelection") except (OSError, ValueError, KeyError, IndexError, TypeError, AttributeError): want = None if want is not None: v = next((e for e in entries if str(e.get("id")) == str(want)), None) else: v = entries[0] if len(entries) == 1 else None if not v or not v.get("productName"): return None return {"id": v.get("id"), "name": v.get("productName", ""), "version": v.get("versionNumber", ""), "build": v.get("buildNumber", "")} def allowed_targets(os_info): """LTSC is always Shop Floor; anything else is Standard or Engineer. Unknown OS: nothing (fail closed).""" if not os_info: return [] if "ltsc" in (os_info.get("name") or "").lower(): return [t for t in config.IMAGE_TYPES if "shopfloor" in t] return [t for t in config.IMAGE_TYPES if "shopfloor" not in t] # --- verification ----------------------------------------------------------- def _sha256(path, progress=None): h = hashlib.sha256() with open(path, "rb") as f: while True: b = f.read(CHUNK) if not b: break h.update(b) if progress: progress(len(b)) return h.hexdigest() def extras(manifest, base=None, disk=None): """Files (and links) in the upload folder that this upload did not send.""" files, links = disk or scan(base) want = {f["path"].lower() for f in manifest["files"]} out = list(links) for low, rels in files.items(): if low not in want: out.extend(rels) elif len(rels) > 1: out.extend(rels[1:]) # same name in two cases: one is stray return sorted(out) def run_verify(manifest, manifest_sha, base=None): """Hash every manifest file on disk; write and return the result. Call with the lock held.""" base = base or root() state_path = os.path.join(base, STATE) cache = {} prev = load_detail() if base == root() else {} if prev.get("upload_id") == manifest["upload_id"]: cache = prev.get("hashes") or {} files = manifest["files"] total_bytes = sum(f["size"] for f in files) st = {"state": "running", "upload_id": manifest["upload_id"], "manifest_sha": manifest_sha, "started": _now(), "total_files": len(files), "total_bytes": total_bytes, "done_files": 0, "done_bytes": 0} _write_json(state_path, st) last = [time.monotonic()] def tick(n): st["done_bytes"] += n if time.monotonic() - last[0] > 2: last[0] = time.monotonic() _write_json(state_path, st) disk = scan(base) missing, mismatch, empty_json, hashes, snap = [], [], [], {}, {} for f in files: rel = f["path"] on_disk = disk[0].get(rel.lower()) or [] if not on_disk: missing.append(rel) st["done_bytes"] += f["size"] st["done_files"] += 1 continue if len(on_disk) > 1: mismatch.append({"path": rel, "reason": "two files differ only in case: " + ", ".join(on_disk)}) st["done_bytes"] += f["size"] st["done_files"] += 1 continue actual = on_disk[0] try: p = safe_join(actual, base) except UploadError: missing.append(rel) st["done_files"] += 1 continue s = os.stat(p) k = _key(s) if s.st_size != f["size"]: mismatch.append({"path": rel, "reason": f"size {s.st_size}, expected {f['size']}"}) st["done_bytes"] += f["size"] else: c = cache.get(actual) if c and c[:4] == k and c[4] == f["sha256"]: sha = c[4] # same file, already proven right st["done_bytes"] += f["size"] else: sha = _sha256(p, tick) if sha != f["sha256"]: mismatch.append({"path": rel, "reason": "content differs"}) else: hashes[actual] = k + [sha] # only proven-good files are cached snap[actual] = k low = rel.lower() if s.st_size == 0 and low.endswith(".json") and low.startswith(("deploy/control/", "tools/")): empty_json.append(rel) st["done_files"] += 1 # Bad at the source: the PC's hash is not the one MCL publishes. bad_cache = [] by_lower = {f["path"].lower(): f for f in files} for low, (_mrel, msha) in mcl_expected(base).items(): f = by_lower.get(low) if f and f["sha256"] != msha: bad_cache.append({"path": f["path"], "reason": "hash differs from MCL's Control JSON"}) for rel in empty_json: bad_cache.append({"path": rel, "reason": "empty Control JSON"}) ext = extras(manifest, base, disk) ok = not (missing or mismatch or bad_cache) os_info = detected_os(manifest, base) if ok else None allowed = allowed_targets(os_info) if ok else [] result = { "state": "done", "ok": ok, "importable": ok and not ext and bool(allowed), "upload_id": manifest["upload_id"], "manifest_sha": manifest_sha, "started": st["started"], "finished": _now(), "total_files": len(files), "total_bytes": total_bytes, "missing": missing, "mismatch": mismatch, "bad_in_cache": bad_cache, "extra": ext[:500], "extra_count": len(ext), "os": os_info, "allowed_targets": allowed, "small_max": SMALL_MAX, } _write_json(os.path.join(base, DETAIL), {"upload_id": manifest["upload_id"], "manifest_sha": manifest_sha, "snapshot": snap, "hashes": hashes}, indent=None) _write_json(state_path, result) return result def start_verify(data): """Store the POSTed manifest and start hashing in the background. Returns the public state.""" m = parse_manifest(data) if not _lock.acquire(blocking=False): raise UploadError("busy: a verification, file upload or import is running", 409) try: base = root() os.makedirs(base, exist_ok=True) raw = json.dumps(data, indent=1).encode("utf-8") mpath = os.path.join(base, MANIFEST) tmp = os.path.join(base, f".{MANIFEST}.part-{secrets.token_hex(8)}") with open(tmp, "wb") as f: f.write(raw) f.flush() os.fsync(f.fileno()) os.replace(tmp, mpath) _chown_like_parent(mpath) msha = hashlib.sha256(raw).hexdigest() _write_json(os.path.join(base, STATE), {"state": "running", "upload_id": m["upload_id"], "manifest_sha": msha, "started": _now(), "total_files": len(m["files"]), "total_bytes": sum(f["size"] for f in m["files"]), "done_files": 0, "done_bytes": 0}) except BaseException: _lock.release() raise def work(): try: run_verify(m, msha) except Exception as ex: # report, never leave "running" behind try: _write_json(os.path.join(root(), STATE), {"state": "error", "ok": False, "importable": False, "upload_id": m["upload_id"], "error": str(ex), "finished": _now()}) except OSError: pass finally: _lock.release() threading.Thread(target=work, name="upload-verify", daemon=True).start() return public_state() def public_state(): s = load_state() return s if s else {"state": "none"} # --- second channel: PUT one small file ------------------------------------- def receive_file(upload_id, rel, stream, length): """Write one manifest file from an HTTP body. Temp name, hash check, rename. Returns {"path", "sha256", "size"}.""" rel = norm_path(rel) with exclusive("file upload"): m, _ = load_manifest() if not m: raise UploadError("no manifest: POST /api/upload/verify first", 409) if upload_id != m["upload_id"]: raise UploadError("upload_id does not match the current manifest", 409) entry = next((f for f in m["files"] if f["path"].lower() == rel.lower()), None) if not entry: raise UploadError(f"not in the manifest: {rel}", 404) if entry["size"] > SMALL_MAX: raise UploadError(f"{entry['path']} is over {SMALL_MAX // (1024 * 1024)} MB: re-copy it with robocopy", 413) if length is None or length != entry["size"]: raise UploadError(f"body is {length} bytes, manifest says {entry['size']}") # Replace the file that is there, whatever its case; else create it, # reusing existing folders case-insensitively (as Samba does). on_disk = scan()[0].get(entry["path"].lower()) or [] if len(on_disk) > 1: raise UploadError(f"two files differ only in case: {', '.join(on_disk)}; remove one", 409) target_rel = on_disk[0] if on_disk else _resolve_dirs(entry["path"]) dest = safe_join(target_rel) d = os.path.dirname(dest) tmp = os.path.join(d, f".{os.path.basename(dest)}.part-{secrets.token_hex(8)}") h, n = hashlib.sha256(), 0 try: with open(tmp, "xb") as f: while n <= entry["size"]: b = stream.read(min(CHUNK, entry["size"] + 1 - n)) if not b: break n += len(b) h.update(b) f.write(b) f.flush() os.fsync(f.fileno()) if n != entry["size"]: raise UploadError(f"received {n} bytes, expected {entry['size']}") if h.hexdigest() != entry["sha256"]: raise UploadError(f"hash of the received {entry['path']} does not match the manifest", 422) if os.path.isdir(dest) and not os.path.islink(dest): raise UploadError(f"a folder is in the way: {target_rel}", 409) _chown_like_parent(tmp) os.chmod(tmp, 0o664) os.replace(tmp, dest) finally: if os.path.exists(tmp): os.remove(tmp) return {"path": entry["path"], "written_as": target_rel, "sha256": entry["sha256"], "size": n} def _resolve_dirs(rel): """Map rel's folders onto existing ones case-insensitively, creating what is missing (refusing links). Returns the path to write.""" cur = os.path.realpath(root()) out = [] for part in rel.split("/")[:-1]: hit = None try: for name in os.listdir(cur): if name.lower() == part.lower(): hit = name break except OSError: pass name = hit or part nxt = os.path.join(cur, name) if os.path.islink(nxt): raise UploadError(f"link in path: {rel}") if not os.path.isdir(nxt): os.mkdir(nxt) _chown_like_parent(nxt) cur = nxt out.append(name) out.append(rel.split("/")[-1]) return "/".join(out) # --- import gate ------------------------------------------------------------ def gate(target=None, locked=False): """(ok, reason, info) for importing the upload folder into target. info carries what the import page shows. Pass locked=True when the caller already holds the lock (the import itself).""" m, msha = load_manifest() info = {"manifest": None, "state": public_state()} if not m: return False, ("No upload manifest. Upload with scripts\\Upload-Image.ps1, which " "checks every file; files copied by hand cannot be verified."), info info["manifest"] = {k: m.get(k) for k in ("upload_id", "created", "source_pc", "mcl")} info["manifest"]["files"] = len(m["files"]) info["manifest"]["bytes"] = sum(f["size"] for f in m["files"]) s = load_state() if (not locked and _lock.locked()) or (s and s.get("state") == "running"): return False, "A verification, file upload or import is running.", info if not s or s.get("state") != "done": return False, "This upload has not been verified yet (run Upload-Image.ps1 to the end).", info if s.get("upload_id") != m["upload_id"] or s.get("manifest_sha") != msha: return False, "The manifest changed after the last verification; verify again.", info if not s.get("ok"): n = len(s.get("missing", [])) + len(s.get("mismatch", [])) + len(s.get("bad_in_cache", [])) return False, f"The last verification found {n} bad file(s).", info det = load_detail() if det.get("upload_id") != m["upload_id"] or det.get("manifest_sha") != msha: return False, "Verification details are missing; verify again.", info snap = det.get("snapshot") or {} if len(snap) != len(m["files"]): return False, "Verification details are incomplete; verify again.", info for rel, key in snap.items(): try: st = os.stat(safe_join(rel)) except (OSError, UploadError): return False, f"{rel} is gone since verification; verify again.", info if _key(st) != key: return False, f"{rel} changed since verification; verify again.", info ext = extras(m) info["extra"] = ext[:50] info["extra_count"] = len(ext) if ext: return False, (f"{len(ext)} file(s) in the upload folder are not part of this upload " "(left from an earlier one). Remove them first."), info allowed = s.get("allowed_targets") or [] info["allowed_targets"] = allowed info["os"] = s.get("os") if not allowed: return False, "Cannot tell which OS this upload carries (OperatingSystem.json), so it has no safe target.", info if target is not None and target not in allowed: return False, f"This upload carries {(s.get('os') or {}).get('name', 'an unknown OS')}; it can only go to: {', '.join(allowed)}.", info return True, "", info def remove_extras(): """Delete files and links in the upload folder that are not in the manifest. Returns how many were removed.""" with exclusive("remove"): m, _ = load_manifest() if not m: raise UploadError("no manifest") n = 0 for rel in extras(m): p = os.path.join(root(), *rel.split("/")) if os.path.islink(p) or os.path.isfile(p): os.remove(p) n += 1 for dp, _dns, _fns in os.walk(root(), topdown=False): if dp != root(): try: os.rmdir(dp) except OSError: pass return n def import_record(target): """What gets stored on the image after a verified import.""" m, _ = load_manifest() s = load_state() or {} return {"upload_id": m["upload_id"] if m else None, "imported": _now(), "target": target, "source_pc": (m or {}).get("source_pc"), "mcl": (m or {}).get("mcl"), "os": s.get("os"), "files": len(m["files"]) if m else 0} def finish_import(image_root, target): """After the import moved the files: keep a record on the image, then clear the manifest and state so the folder starts fresh. Call with the lock held.""" rec = import_record(target) try: _write_json(os.path.join(image_root, ".mcl-import.json"), rec) except OSError: pass for name in TOP_FILES: try: os.remove(os.path.join(root(), name)) except OSError: pass return rec def last_import(image_root): try: return _read_json(os.path.join(image_root, ".mcl-import.json")) except (OSError, ValueError): return None def try_lock(): """For the import route: take the lock without waiting (True if held).""" return _lock.acquire(blocking=False) def unlock(): _lock.release()