Verified MCL uploads: hash every file, repair over HTTP, gate the import

Small Control JSONs have arrived empty or wrong more than once, and the old
check compared sizes only and skipped missing files. A copy of
Deploy/Control on the dev PC still holds two 0-byte JSONs.

Upload-Image.ps1 (tested on Windows PowerShell 5.1 in the win11 VM):
- refuses an unfinished MCL cache: exactly one <OS>-Production.tag (MCL
  writes it last), no Control file newer than it, selected OS matches,
  every Control/Tools JSON non-empty and valid
- checks payload against the SHA-256 MCL's own Control JSONs publish. MCL
  resumes downloads onto old files (FileMode.Append) and then trusts any
  file of the right size: the September junk-tail corruption. Offers to
  delete the bad cache files so MCL downloads them again
- hashes every file into a manifest, robocopies, POSTs the manifest
- re-sends bad small files over HTTP (no proxy), large ones by deleting
  the bad copy and robocopy /J (robocopy skips a same-size, same-time file
  as "modified" even with /IS /IT)

Webapp:
- services/upload_verify.py hashes the upload on the server's own disk in
  a background thread; reports missing, differing, bad-in-MCL-cache and
  stray files; PUT /api/upload/file writes temp, checks hash, renames
- import of the upload folder only after a passed verification, with
  every file unchanged (size, mtime, ctime, inode), no stray files, and
  the target the OS allows (LTSC: Shop Floor only; else Standard or
  Engineer; unknown OS: none). Import, verify and PUT share one lock
- paths matched case-insensitively against the disk, as Samba does
- import page shows the upload, MCL run, verification and stray files
  with a remove button; record of each import kept on the image

Reviewed adversarially; all 11 confirmed findings fixed with tests.
42 new unit tests. VM test: clean upload; same-size same-mtime damage on
3 files repaired in one round; stray file blocks import until removed;
Shop Floor refused for a Pro upload; import records and clears the
folder; missing tag, 0-byte JSON, junk-head WIM and late Control file
each refused with nothing uploaded.

webapp/app.py

@@ -60,6 +60,7 @@ from services import (
     nic_status,
     system,
     unattend,
+    upload_verify,
     wim,
 )
 from services.audit import audit
@@ -217,11 +218,32 @@ def images_import():
             flash(f"Source path does not exist: {source}", "danger")
             return redirect(url_for("images_import"))
 
+        # Network uploads must be the verified Upload-Image.ps1 upload: every
+        # file hashed on this disk against the PC's manifest, nothing stray.
+        up_real = os.path.realpath(config.UPLOAD_DIR)
+        src_real = os.path.realpath(source)
+        is_upload = src_real == up_real or src_real.startswith(up_real + "/")
+        if is_upload:
+            if src_real != up_real:
+                flash("Import the whole upload folder; its manifest covers all of it.", "danger")
+                return redirect(url_for("images_import"))
+            # Held through gate, move and clean-up: no verify or PUT may
+            # touch the folder while its files are being moved.
+            if not upload_verify.try_lock():
+                flash("Import refused: a verification or file upload is running. Try again.", "danger")
+                return redirect(url_for("images_import"))
+            ok, reason, _info = upload_verify.gate(target, locked=True)
+            if not ok:
+                upload_verify.unlock()
+                flash(f"Import refused: {reason}", "danger")
+                return redirect(url_for("images_import"))
+
         root = fs.image_root(target)
         dest = fs.deploy_path(target)
         try:
             os.makedirs(dest, exist_ok=True)
-            src_items = os.listdir(source)
+            # The manifest / verify state stay behind (finish_import clears them).
+            src_items = [i for i in os.listdir(source) if not upload_verify.is_reserved(i)]
 
             # Move files from network upload to save disk space; copy from USB.
             # deploy.sync_tree uses rsync --checksum: unchanged files are
@@ -265,18 +287,32 @@ def images_import():
             media_tag = os.path.join(control_dir, "Media.tag")
             Path(media_tag).touch()
 
-            audit("IMAGE_IMPORT", f"{source} -> {target}")
+            if is_upload:
+                rec = upload_verify.finish_import(root, target)
+                audit("IMAGE_IMPORT", f"{source} -> {target} (verified upload {rec.get('upload_id')})")
+            else:
+                audit("IMAGE_IMPORT", f"{source} -> {target} (USB, not verified)")
             flash(
                 f"Successfully imported content to {config.FRIENDLY_NAMES.get(target, target)}.",
                 "success",
             )
         except Exception as exc:
             flash(f"Import failed: {exc}", "danger")
+        finally:
+            if is_upload:
+                upload_verify.unlock()
 
         return redirect(url_for("images_import"))
 
+    up_ok, up_reason, up_info = upload_verify.gate()
+    last_imports = {it: upload_verify.last_import(fs.image_root(it)) for it in config.IMAGE_TYPES}
     return render_template(
         "import.html",
+        upload_ok=up_ok,
+        upload_reason=up_reason,
+        upload_info=up_info,
+        last_imports=last_imports,
+        upload_dir=config.UPLOAD_DIR,
         usb_mounts=usb_mounts,
         upload_sources=upload_sources,
         images=image_list,
@@ -1872,6 +1908,46 @@ def audit_log():
 # Routes - JSON API
 # ---------------------------------------------------------------------------
 
+@app.route("/images/import/remove-extras", methods=["POST"])
+def images_import_remove_extras():
+    try:
+        n = upload_verify.remove_extras()
+    except upload_verify.UploadError as ex:
+        flash(f"Could not remove: {ex}", "danger")
+        return redirect(url_for("images_import"))
+    audit("UPLOAD_REMOVE_EXTRAS", f"{n} file(s) not in the manifest removed")
+    flash(f"Removed {n} file(s) that were not part of this upload.", "success")
+    return redirect(url_for("images_import"))
+
+
+@app.route("/api/upload/verify", methods=["GET", "POST"])
+def api_upload_verify():
+    # CSRF-exempt machine-to-machine endpoint: Upload-Image.ps1 POSTs its
+    # manifest after robocopy, then polls GET until the hashing is done.
+    if request.method == "GET":
+        return jsonify(upload_verify.public_state())
+    data = request.get_json(silent=True)
+    try:
+        st = upload_verify.start_verify(data)
+    except upload_verify.UploadError as ex:
+        return jsonify({"error": str(ex)}), ex.status
+    audit("UPLOAD_VERIFY", f"upload {st.get('upload_id')}: {st.get('total_files')} file(s)")
+    return jsonify(st), 202
+
+
+@app.route("/api/upload/file", methods=["PUT"])
+def api_upload_file():
+    # Second channel for files SMB delivered wrong: one manifest file per
+    # PUT, hash-checked before it replaces anything.
+    try:
+        res = upload_verify.receive_file(request.args.get("upload_id", ""), request.args.get("path", ""),
+                                         request.stream, request.content_length)
+    except upload_verify.UploadError as ex:
+        return jsonify({"error": str(ex)}), ex.status
+    audit("UPLOAD_FILE_RESENT", f"{res['path']} ({res['size']} bytes) over HTTP")
+    return jsonify(res), 200
+
+
 @app.route("/api/services")
 def api_services():
     services = {s: system.service_status(s) for s in ("dnsmasq", "apache2", "smbd")}