bca04f4ca2
Small Control JSONs have arrived empty or wrong more than once, and the old check compared sizes only and skipped missing files. A copy of Deploy/Control on the dev PC still holds two 0-byte JSONs. Upload-Image.ps1 (tested on Windows PowerShell 5.1 in the win11 VM): - refuses an unfinished MCL cache: exactly one <OS>-Production.tag (MCL writes it last), no Control file newer than it, selected OS matches, every Control/Tools JSON non-empty and valid - checks payload against the SHA-256 MCL's own Control JSONs publish. MCL resumes downloads onto old files (FileMode.Append) and then trusts any file of the right size: the September junk-tail corruption. Offers to delete the bad cache files so MCL downloads them again - hashes every file into a manifest, robocopies, POSTs the manifest - re-sends bad small files over HTTP (no proxy), large ones by deleting the bad copy and robocopy /J (robocopy skips a same-size, same-time file as "modified" even with /IS /IT) Webapp: - services/upload_verify.py hashes the upload on the server's own disk in a background thread; reports missing, differing, bad-in-MCL-cache and stray files; PUT /api/upload/file writes temp, checks hash, renames - import of the upload folder only after a passed verification, with every file unchanged (size, mtime, ctime, inode), no stray files, and the target the OS allows (LTSC: Shop Floor only; else Standard or Engineer; unknown OS: none). Import, verify and PUT share one lock - paths matched case-insensitively against the disk, as Samba does - import page shows the upload, MCL run, verification and stray files with a remove button; record of each import kept on the image Reviewed adversarially; all 11 confirmed findings fixed with tests. 42 new unit tests. VM test: clean upload; same-size same-mtime damage on 3 files repaired in one round; stray file blocks import until removed; Shop Floor refused for a Pro upload; import records and clears the folder; missing tag, 0-byte JSON, junk-head WIM and late Control file each refused with nothing uploaded.
709 lines
27 KiB
Python
709 lines
27 KiB
Python
"""Verified MCL uploads: prove every file in the upload folder is the one
|
|
the Windows PC sent, repair bad ones over HTTP, and gate the import.
|
|
|
|
Upload-Image.ps1 (work PC) hashes every file it copies into
|
|
upload-manifest.json and POSTs that manifest here after robocopy. A
|
|
background thread then hashes the files on THIS disk (no SMB client cache
|
|
in the way) and records which are missing, which differ, and which were
|
|
already bad in the MCL cache (their hash on the PC does not match the hash
|
|
MCL's own Control JSONs publish). The script re-sends the bad small files
|
|
with PUT, which writes them to a temp name, checks the hash and renames
|
|
them into place.
|
|
|
|
The import page only imports an upload folder whose last verification
|
|
passed and whose files have not changed since (size, mtime, ctime and
|
|
inode of every file), with no stray files from an earlier upload mixed in.
|
|
The import holds the same lock as verification and PUT, so neither can
|
|
run while files are being moved.
|
|
|
|
Paths are matched against the disk case-insensitively, as Windows and
|
|
Samba do: the manifest's case comes from MCL's JSON, the disk's from
|
|
whatever robocopy created first.
|
|
|
|
State files live at the top of the upload folder and are never imported:
|
|
upload-manifest.json the manifest (written by the POST, not by SMB)
|
|
.upload-verify.json progress / last result (small; polled)
|
|
.upload-verify-files.json per-file hash cache and snapshot (gate only)
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import threading
|
|
import time
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
|
|
import config
|
|
|
|
MANIFEST = "upload-manifest.json"
|
|
STATE = ".upload-verify.json"
|
|
DETAIL = ".upload-verify-files.json"
|
|
TOP_FILES = (MANIFEST, STATE, DETAIL)
|
|
PART_RE = re.compile(r"^\..+\.part-[0-9a-f]{16}$")
|
|
|
|
SMALL_MAX = 64 * 1024 * 1024 # PUT limit; bigger files go back to robocopy
|
|
MAX_FILES = 200_000
|
|
UPLOAD_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,80}$")
|
|
SHA_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
|
DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
|
CHUNK = 1024 * 1024
|
|
|
|
_lock = threading.Lock() # one verify, PUT or import at a time
|
|
|
|
|
|
class UploadError(ValueError):
|
|
"""Bad request (400) unless .status says otherwise."""
|
|
|
|
def __init__(self, msg, status=400):
|
|
super().__init__(msg)
|
|
self.status = status
|
|
|
|
|
|
@contextmanager
|
|
def exclusive(what):
|
|
"""Hold the upload lock or fail at once with 409."""
|
|
if not _lock.acquire(blocking=False):
|
|
raise UploadError(f"busy: a verification, file upload or import is running ({what} refused)", 409)
|
|
try:
|
|
yield
|
|
finally:
|
|
_lock.release()
|
|
|
|
|
|
# --- paths ------------------------------------------------------------------
|
|
|
|
def root():
|
|
return config.UPLOAD_DIR
|
|
|
|
|
|
def _now():
|
|
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
|
|
|
|
|
def is_reserved(rel):
|
|
"""Only our own files at the top of the folder (and their temp files).
|
|
A '.part-' file deeper down is debris and counts as stray."""
|
|
if "/" in rel:
|
|
return False
|
|
if rel in TOP_FILES:
|
|
return True
|
|
m = PART_RE.match(rel)
|
|
return bool(m) and any(rel.startswith(f".{n}.part-") for n in TOP_FILES)
|
|
|
|
|
|
def norm_path(p):
|
|
"""Manifest path -> 'Deploy/Control/x.json'. Raises on anything that
|
|
could leave the upload folder."""
|
|
if not isinstance(p, str) or not p or len(p) > 1024 or "\x00" in p:
|
|
raise UploadError("bad path")
|
|
p = p.replace("\\", "/")
|
|
if p.startswith("/") or DRIVE_RE.match(p) or ":" in p:
|
|
raise UploadError(f"absolute path not allowed: {p}")
|
|
parts = p.split("/")
|
|
if any(s in ("", ".", "..") for s in parts):
|
|
raise UploadError(f"bad path: {p}")
|
|
if is_reserved(p) or PART_RE.match(parts[-1]):
|
|
raise UploadError(f"reserved name: {p}")
|
|
return p
|
|
|
|
|
|
def safe_join(rel, base=None):
|
|
"""Absolute path of rel inside base, refusing symlink escapes."""
|
|
base = os.path.realpath(base or root())
|
|
full = os.path.join(base, *rel.split("/"))
|
|
real = os.path.realpath(full)
|
|
if real != full or not real.startswith(base + os.sep):
|
|
# realpath differs: a symlink somewhere on the way. Refuse it.
|
|
raise UploadError(f"path leaves the upload folder or uses a link: {rel}")
|
|
return full
|
|
|
|
|
|
def scan(base=None):
|
|
"""Walk the upload folder once.
|
|
Returns (files, links): files = {rel_lower: [rel, ...]} for regular
|
|
files; links = rels of symlinks and other non-regular entries, which
|
|
are never part of an upload."""
|
|
base = base or root()
|
|
files, links = {}, []
|
|
for dp, dns, fns in os.walk(base):
|
|
for d in list(dns):
|
|
if os.path.islink(os.path.join(dp, d)):
|
|
links.append(os.path.relpath(os.path.join(dp, d), base).replace(os.sep, "/"))
|
|
dns.remove(d)
|
|
dns.sort()
|
|
for fn in sorted(fns):
|
|
full = os.path.join(dp, fn)
|
|
rel = os.path.relpath(full, base).replace(os.sep, "/")
|
|
if is_reserved(rel):
|
|
continue
|
|
if os.path.islink(full) or not os.path.isfile(full):
|
|
links.append(rel)
|
|
continue
|
|
files.setdefault(rel.lower(), []).append(rel)
|
|
return files, links
|
|
|
|
|
|
def _key(st):
|
|
"""What 'unchanged' means: robocopy keeps the source mtime, so mtime and
|
|
size alone cannot tell a re-copy from the bad copy before it. ctime is
|
|
set by the kernel on every write and cannot be set from userland."""
|
|
return [st.st_size, st.st_mtime_ns, st.st_ctime_ns, st.st_ino]
|
|
|
|
|
|
# --- manifest ---------------------------------------------------------------
|
|
|
|
def parse_manifest(data):
|
|
"""Validate a manifest dict; return it with normalised paths and
|
|
lower-case hashes."""
|
|
if not isinstance(data, dict):
|
|
raise UploadError("manifest must be a JSON object")
|
|
if data.get("manifest_version") != 1:
|
|
raise UploadError("unsupported manifest_version (want 1)")
|
|
uid = data.get("upload_id")
|
|
if not isinstance(uid, str) or not UPLOAD_ID_RE.match(uid):
|
|
raise UploadError("bad upload_id")
|
|
files = data.get("files")
|
|
if not isinstance(files, list) or not files:
|
|
raise UploadError("manifest has no files")
|
|
if len(files) > MAX_FILES:
|
|
raise UploadError("too many files")
|
|
out, seen = [], set()
|
|
for f in files:
|
|
if not isinstance(f, dict):
|
|
raise UploadError("bad file entry")
|
|
p = norm_path(f.get("path"))
|
|
size, sha = f.get("size"), f.get("sha256")
|
|
if not isinstance(size, int) or isinstance(size, bool) or size < 0:
|
|
raise UploadError(f"bad size for {p}")
|
|
if not isinstance(sha, str) or not SHA_RE.match(sha):
|
|
raise UploadError(f"bad sha256 for {p}")
|
|
if p.lower() in seen:
|
|
raise UploadError(f"duplicate path: {p}")
|
|
seen.add(p.lower())
|
|
out.append({"path": p, "size": size, "sha256": sha.lower()})
|
|
mcl = data.get("mcl") if isinstance(data.get("mcl"), dict) else {}
|
|
clean = {"manifest_version": 1, "upload_id": uid, "files": out, "mcl": mcl}
|
|
for k in ("created", "source_pc"):
|
|
if isinstance(data.get(k), str):
|
|
clean[k] = data[k][:200]
|
|
return clean
|
|
|
|
|
|
def _read_json(path):
|
|
with open(path, "rb") as f:
|
|
return json.loads(f.read().decode("utf-8-sig"))
|
|
|
|
|
|
def _write_json(path, obj, indent=1):
|
|
"""Atomic: temp file in the same folder, fsync, rename."""
|
|
d = os.path.dirname(path)
|
|
tmp = os.path.join(d, f".{os.path.basename(path)}.part-{secrets.token_hex(8)}")
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
json.dump(obj, f, indent=indent)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def load_manifest():
|
|
"""(manifest, raw_sha) or (None, None) when there is none."""
|
|
p = os.path.join(root(), MANIFEST)
|
|
if not os.path.isfile(p):
|
|
return None, None
|
|
with open(p, "rb") as f:
|
|
raw = f.read()
|
|
try:
|
|
m = parse_manifest(json.loads(raw.decode("utf-8-sig")))
|
|
except (ValueError, UnicodeDecodeError):
|
|
return None, None
|
|
return m, hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def load_state():
|
|
try:
|
|
return _read_json(os.path.join(root(), STATE))
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def load_detail():
|
|
try:
|
|
return _read_json(os.path.join(root(), DETAIL))
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def _chown_like_parent(path):
|
|
"""Match the share's owner (Samba 'force user = pxe') so a later SMB
|
|
re-copy can overwrite what the webapp (root) wrote."""
|
|
try:
|
|
st = os.stat(os.path.dirname(path))
|
|
os.chown(path, st.st_uid, st.st_gid)
|
|
except (OSError, AttributeError):
|
|
pass
|
|
|
|
|
|
# --- MCL's own hashes and OS ------------------------------------------------
|
|
|
|
def _mcl_rel(dest_dir, file_name):
|
|
"""MCL destination ('*destinationdir*\\Deploy\\Packages\\X' or
|
|
'C:\\Deploy\\Languages\\Y') + file name -> 'Deploy/Packages/X/name'."""
|
|
d = (dest_dir or "").replace("\\", "/")
|
|
d = re.sub(r"^\*destinationdir\*/?", "", d, flags=re.I)
|
|
d = re.sub(r"^[A-Za-z]:/", "", d).strip("/")
|
|
return f"{d}/{file_name}" if d else file_name
|
|
|
|
|
|
def _control_list(base, name):
|
|
try:
|
|
d = _read_json(os.path.join(base, "Deploy", "Control", name))
|
|
return d if isinstance(d, list) else []
|
|
except (OSError, ValueError, UnicodeDecodeError):
|
|
return []
|
|
|
|
|
|
def mcl_expected(base=None):
|
|
"""{path_lower: (path, sha256_lower)} for every payload file the
|
|
Control JSONs in this upload describe. Only real SHA-256 values."""
|
|
base = base or root()
|
|
exp = {}
|
|
|
|
def add(dest, name, sha):
|
|
if not (dest and name and isinstance(sha, str) and SHA_RE.match(sha)):
|
|
return
|
|
rel = _mcl_rel(dest, name)
|
|
exp[rel.lower()] = (rel, sha.lower())
|
|
|
|
for e in _control_list(base, "OperatingSystem.json"):
|
|
wim = ((e or {}).get("operatingSystemVersion") or {}).get("wim") or {}
|
|
add(wim.get("destinationDir") or wim.get("DestinationDir"),
|
|
wim.get("fileName") or wim.get("FileName") or "install.wim", wim.get("hash") or wim.get("Hash"))
|
|
for fname in ("HardwareDriver.json", "packages.json"):
|
|
for e in _control_list(base, fname):
|
|
if isinstance(e, dict):
|
|
add(e.get("destinationDir") or e.get("DestinationDir"),
|
|
e.get("fileName") or e.get("FileName"), e.get("hash") or e.get("Hash"))
|
|
for e in _control_list(base, "LanguagePacks.json"):
|
|
for lf in (e or {}).get("files") or []:
|
|
if isinstance(lf, dict):
|
|
add(lf.get("destFolder"), lf.get("fileName"), lf.get("hash"))
|
|
return exp
|
|
|
|
|
|
def detected_os(manifest=None, base=None):
|
|
"""The OS the upload carries: the OperatingSystem.json entry whose id
|
|
is the selected one (manifest mcl.os_id, else Tools/user_selections.json).
|
|
A single entry is taken as is. None when it cannot be told."""
|
|
base = base or root()
|
|
entries = [e.get("operatingSystemVersion") for e in _control_list(base, "OperatingSystem.json")
|
|
if isinstance(e, dict) and isinstance(e.get("operatingSystemVersion"), dict)]
|
|
if not entries:
|
|
return None
|
|
want = ((manifest or {}).get("mcl") or {}).get("os_id")
|
|
if want is None:
|
|
try:
|
|
sel = _read_json(os.path.join(base, "Tools", "user_selections.json"))
|
|
want = sel[0].get("OperatingSystemSelection")
|
|
except (OSError, ValueError, KeyError, IndexError, TypeError, AttributeError):
|
|
want = None
|
|
if want is not None:
|
|
v = next((e for e in entries if str(e.get("id")) == str(want)), None)
|
|
else:
|
|
v = entries[0] if len(entries) == 1 else None
|
|
if not v or not v.get("productName"):
|
|
return None
|
|
return {"id": v.get("id"), "name": v.get("productName", ""),
|
|
"version": v.get("versionNumber", ""), "build": v.get("buildNumber", "")}
|
|
|
|
|
|
def allowed_targets(os_info):
|
|
"""LTSC is always Shop Floor; anything else is Standard or Engineer.
|
|
Unknown OS: nothing (fail closed)."""
|
|
if not os_info:
|
|
return []
|
|
if "ltsc" in (os_info.get("name") or "").lower():
|
|
return [t for t in config.IMAGE_TYPES if "shopfloor" in t]
|
|
return [t for t in config.IMAGE_TYPES if "shopfloor" not in t]
|
|
|
|
|
|
# --- verification -----------------------------------------------------------
|
|
|
|
def _sha256(path, progress=None):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
while True:
|
|
b = f.read(CHUNK)
|
|
if not b:
|
|
break
|
|
h.update(b)
|
|
if progress:
|
|
progress(len(b))
|
|
return h.hexdigest()
|
|
|
|
|
|
def extras(manifest, base=None, disk=None):
|
|
"""Files (and links) in the upload folder that this upload did not send."""
|
|
files, links = disk or scan(base)
|
|
want = {f["path"].lower() for f in manifest["files"]}
|
|
out = list(links)
|
|
for low, rels in files.items():
|
|
if low not in want:
|
|
out.extend(rels)
|
|
elif len(rels) > 1:
|
|
out.extend(rels[1:]) # same name in two cases: one is stray
|
|
return sorted(out)
|
|
|
|
|
|
def run_verify(manifest, manifest_sha, base=None):
|
|
"""Hash every manifest file on disk; write and return the result.
|
|
Call with the lock held."""
|
|
base = base or root()
|
|
state_path = os.path.join(base, STATE)
|
|
cache = {}
|
|
prev = load_detail() if base == root() else {}
|
|
if prev.get("upload_id") == manifest["upload_id"]:
|
|
cache = prev.get("hashes") or {}
|
|
|
|
files = manifest["files"]
|
|
total_bytes = sum(f["size"] for f in files)
|
|
st = {"state": "running", "upload_id": manifest["upload_id"], "manifest_sha": manifest_sha,
|
|
"started": _now(), "total_files": len(files), "total_bytes": total_bytes,
|
|
"done_files": 0, "done_bytes": 0}
|
|
_write_json(state_path, st)
|
|
last = [time.monotonic()]
|
|
|
|
def tick(n):
|
|
st["done_bytes"] += n
|
|
if time.monotonic() - last[0] > 2:
|
|
last[0] = time.monotonic()
|
|
_write_json(state_path, st)
|
|
|
|
disk = scan(base)
|
|
missing, mismatch, empty_json, hashes, snap = [], [], [], {}, {}
|
|
for f in files:
|
|
rel = f["path"]
|
|
on_disk = disk[0].get(rel.lower()) or []
|
|
if not on_disk:
|
|
missing.append(rel)
|
|
st["done_bytes"] += f["size"]
|
|
st["done_files"] += 1
|
|
continue
|
|
if len(on_disk) > 1:
|
|
mismatch.append({"path": rel, "reason": "two files differ only in case: " + ", ".join(on_disk)})
|
|
st["done_bytes"] += f["size"]
|
|
st["done_files"] += 1
|
|
continue
|
|
actual = on_disk[0]
|
|
try:
|
|
p = safe_join(actual, base)
|
|
except UploadError:
|
|
missing.append(rel)
|
|
st["done_files"] += 1
|
|
continue
|
|
s = os.stat(p)
|
|
k = _key(s)
|
|
if s.st_size != f["size"]:
|
|
mismatch.append({"path": rel, "reason": f"size {s.st_size}, expected {f['size']}"})
|
|
st["done_bytes"] += f["size"]
|
|
else:
|
|
c = cache.get(actual)
|
|
if c and c[:4] == k and c[4] == f["sha256"]:
|
|
sha = c[4] # same file, already proven right
|
|
st["done_bytes"] += f["size"]
|
|
else:
|
|
sha = _sha256(p, tick)
|
|
if sha != f["sha256"]:
|
|
mismatch.append({"path": rel, "reason": "content differs"})
|
|
else:
|
|
hashes[actual] = k + [sha] # only proven-good files are cached
|
|
snap[actual] = k
|
|
low = rel.lower()
|
|
if s.st_size == 0 and low.endswith(".json") and low.startswith(("deploy/control/", "tools/")):
|
|
empty_json.append(rel)
|
|
st["done_files"] += 1
|
|
|
|
# Bad at the source: the PC's hash is not the one MCL publishes.
|
|
bad_cache = []
|
|
by_lower = {f["path"].lower(): f for f in files}
|
|
for low, (_mrel, msha) in mcl_expected(base).items():
|
|
f = by_lower.get(low)
|
|
if f and f["sha256"] != msha:
|
|
bad_cache.append({"path": f["path"], "reason": "hash differs from MCL's Control JSON"})
|
|
for rel in empty_json:
|
|
bad_cache.append({"path": rel, "reason": "empty Control JSON"})
|
|
|
|
ext = extras(manifest, base, disk)
|
|
ok = not (missing or mismatch or bad_cache)
|
|
os_info = detected_os(manifest, base) if ok else None
|
|
allowed = allowed_targets(os_info) if ok else []
|
|
result = {
|
|
"state": "done", "ok": ok, "importable": ok and not ext and bool(allowed),
|
|
"upload_id": manifest["upload_id"], "manifest_sha": manifest_sha,
|
|
"started": st["started"], "finished": _now(),
|
|
"total_files": len(files), "total_bytes": total_bytes,
|
|
"missing": missing,
|
|
"mismatch": mismatch,
|
|
"bad_in_cache": bad_cache,
|
|
"extra": ext[:500], "extra_count": len(ext),
|
|
"os": os_info,
|
|
"allowed_targets": allowed,
|
|
"small_max": SMALL_MAX,
|
|
}
|
|
_write_json(os.path.join(base, DETAIL), {"upload_id": manifest["upload_id"],
|
|
"manifest_sha": manifest_sha,
|
|
"snapshot": snap, "hashes": hashes}, indent=None)
|
|
_write_json(state_path, result)
|
|
return result
|
|
|
|
|
|
def start_verify(data):
|
|
"""Store the POSTed manifest and start hashing in the background.
|
|
Returns the public state."""
|
|
m = parse_manifest(data)
|
|
if not _lock.acquire(blocking=False):
|
|
raise UploadError("busy: a verification, file upload or import is running", 409)
|
|
try:
|
|
base = root()
|
|
os.makedirs(base, exist_ok=True)
|
|
raw = json.dumps(data, indent=1).encode("utf-8")
|
|
mpath = os.path.join(base, MANIFEST)
|
|
tmp = os.path.join(base, f".{MANIFEST}.part-{secrets.token_hex(8)}")
|
|
with open(tmp, "wb") as f:
|
|
f.write(raw)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.replace(tmp, mpath)
|
|
_chown_like_parent(mpath)
|
|
msha = hashlib.sha256(raw).hexdigest()
|
|
_write_json(os.path.join(base, STATE),
|
|
{"state": "running", "upload_id": m["upload_id"], "manifest_sha": msha,
|
|
"started": _now(), "total_files": len(m["files"]),
|
|
"total_bytes": sum(f["size"] for f in m["files"]), "done_files": 0,
|
|
"done_bytes": 0})
|
|
except BaseException:
|
|
_lock.release()
|
|
raise
|
|
|
|
def work():
|
|
try:
|
|
run_verify(m, msha)
|
|
except Exception as ex: # report, never leave "running" behind
|
|
try:
|
|
_write_json(os.path.join(root(), STATE),
|
|
{"state": "error", "ok": False, "importable": False,
|
|
"upload_id": m["upload_id"], "error": str(ex), "finished": _now()})
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
_lock.release()
|
|
|
|
threading.Thread(target=work, name="upload-verify", daemon=True).start()
|
|
return public_state()
|
|
|
|
|
|
def public_state():
|
|
s = load_state()
|
|
return s if s else {"state": "none"}
|
|
|
|
|
|
# --- second channel: PUT one small file -------------------------------------
|
|
|
|
def receive_file(upload_id, rel, stream, length):
|
|
"""Write one manifest file from an HTTP body. Temp name, hash check,
|
|
rename. Returns {"path", "sha256", "size"}."""
|
|
rel = norm_path(rel)
|
|
with exclusive("file upload"):
|
|
m, _ = load_manifest()
|
|
if not m:
|
|
raise UploadError("no manifest: POST /api/upload/verify first", 409)
|
|
if upload_id != m["upload_id"]:
|
|
raise UploadError("upload_id does not match the current manifest", 409)
|
|
entry = next((f for f in m["files"] if f["path"].lower() == rel.lower()), None)
|
|
if not entry:
|
|
raise UploadError(f"not in the manifest: {rel}", 404)
|
|
if entry["size"] > SMALL_MAX:
|
|
raise UploadError(f"{entry['path']} is over {SMALL_MAX // (1024 * 1024)} MB: re-copy it with robocopy", 413)
|
|
if length is None or length != entry["size"]:
|
|
raise UploadError(f"body is {length} bytes, manifest says {entry['size']}")
|
|
# Replace the file that is there, whatever its case; else create it,
|
|
# reusing existing folders case-insensitively (as Samba does).
|
|
on_disk = scan()[0].get(entry["path"].lower()) or []
|
|
if len(on_disk) > 1:
|
|
raise UploadError(f"two files differ only in case: {', '.join(on_disk)}; remove one", 409)
|
|
target_rel = on_disk[0] if on_disk else _resolve_dirs(entry["path"])
|
|
dest = safe_join(target_rel)
|
|
d = os.path.dirname(dest)
|
|
tmp = os.path.join(d, f".{os.path.basename(dest)}.part-{secrets.token_hex(8)}")
|
|
h, n = hashlib.sha256(), 0
|
|
try:
|
|
with open(tmp, "xb") as f:
|
|
while n <= entry["size"]:
|
|
b = stream.read(min(CHUNK, entry["size"] + 1 - n))
|
|
if not b:
|
|
break
|
|
n += len(b)
|
|
h.update(b)
|
|
f.write(b)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
if n != entry["size"]:
|
|
raise UploadError(f"received {n} bytes, expected {entry['size']}")
|
|
if h.hexdigest() != entry["sha256"]:
|
|
raise UploadError(f"hash of the received {entry['path']} does not match the manifest", 422)
|
|
if os.path.isdir(dest) and not os.path.islink(dest):
|
|
raise UploadError(f"a folder is in the way: {target_rel}", 409)
|
|
_chown_like_parent(tmp)
|
|
os.chmod(tmp, 0o664)
|
|
os.replace(tmp, dest)
|
|
finally:
|
|
if os.path.exists(tmp):
|
|
os.remove(tmp)
|
|
return {"path": entry["path"], "written_as": target_rel, "sha256": entry["sha256"], "size": n}
|
|
|
|
|
|
def _resolve_dirs(rel):
|
|
"""Map rel's folders onto existing ones case-insensitively, creating
|
|
what is missing (refusing links). Returns the path to write."""
|
|
cur = os.path.realpath(root())
|
|
out = []
|
|
for part in rel.split("/")[:-1]:
|
|
hit = None
|
|
try:
|
|
for name in os.listdir(cur):
|
|
if name.lower() == part.lower():
|
|
hit = name
|
|
break
|
|
except OSError:
|
|
pass
|
|
name = hit or part
|
|
nxt = os.path.join(cur, name)
|
|
if os.path.islink(nxt):
|
|
raise UploadError(f"link in path: {rel}")
|
|
if not os.path.isdir(nxt):
|
|
os.mkdir(nxt)
|
|
_chown_like_parent(nxt)
|
|
cur = nxt
|
|
out.append(name)
|
|
out.append(rel.split("/")[-1])
|
|
return "/".join(out)
|
|
|
|
|
|
# --- import gate ------------------------------------------------------------
|
|
|
|
def gate(target=None, locked=False):
|
|
"""(ok, reason, info) for importing the upload folder into target.
|
|
info carries what the import page shows. Pass locked=True when the
|
|
caller already holds the lock (the import itself)."""
|
|
m, msha = load_manifest()
|
|
info = {"manifest": None, "state": public_state()}
|
|
if not m:
|
|
return False, ("No upload manifest. Upload with scripts\\Upload-Image.ps1, which "
|
|
"checks every file; files copied by hand cannot be verified."), info
|
|
info["manifest"] = {k: m.get(k) for k in ("upload_id", "created", "source_pc", "mcl")}
|
|
info["manifest"]["files"] = len(m["files"])
|
|
info["manifest"]["bytes"] = sum(f["size"] for f in m["files"])
|
|
s = load_state()
|
|
if (not locked and _lock.locked()) or (s and s.get("state") == "running"):
|
|
return False, "A verification, file upload or import is running.", info
|
|
if not s or s.get("state") != "done":
|
|
return False, "This upload has not been verified yet (run Upload-Image.ps1 to the end).", info
|
|
if s.get("upload_id") != m["upload_id"] or s.get("manifest_sha") != msha:
|
|
return False, "The manifest changed after the last verification; verify again.", info
|
|
if not s.get("ok"):
|
|
n = len(s.get("missing", [])) + len(s.get("mismatch", [])) + len(s.get("bad_in_cache", []))
|
|
return False, f"The last verification found {n} bad file(s).", info
|
|
det = load_detail()
|
|
if det.get("upload_id") != m["upload_id"] or det.get("manifest_sha") != msha:
|
|
return False, "Verification details are missing; verify again.", info
|
|
snap = det.get("snapshot") or {}
|
|
if len(snap) != len(m["files"]):
|
|
return False, "Verification details are incomplete; verify again.", info
|
|
for rel, key in snap.items():
|
|
try:
|
|
st = os.stat(safe_join(rel))
|
|
except (OSError, UploadError):
|
|
return False, f"{rel} is gone since verification; verify again.", info
|
|
if _key(st) != key:
|
|
return False, f"{rel} changed since verification; verify again.", info
|
|
ext = extras(m)
|
|
info["extra"] = ext[:50]
|
|
info["extra_count"] = len(ext)
|
|
if ext:
|
|
return False, (f"{len(ext)} file(s) in the upload folder are not part of this upload "
|
|
"(left from an earlier one). Remove them first."), info
|
|
allowed = s.get("allowed_targets") or []
|
|
info["allowed_targets"] = allowed
|
|
info["os"] = s.get("os")
|
|
if not allowed:
|
|
return False, "Cannot tell which OS this upload carries (OperatingSystem.json), so it has no safe target.", info
|
|
if target is not None and target not in allowed:
|
|
return False, f"This upload carries {(s.get('os') or {}).get('name', 'an unknown OS')}; it can only go to: {', '.join(allowed)}.", info
|
|
return True, "", info
|
|
|
|
|
|
def remove_extras():
|
|
"""Delete files and links in the upload folder that are not in the
|
|
manifest. Returns how many were removed."""
|
|
with exclusive("remove"):
|
|
m, _ = load_manifest()
|
|
if not m:
|
|
raise UploadError("no manifest")
|
|
n = 0
|
|
for rel in extras(m):
|
|
p = os.path.join(root(), *rel.split("/"))
|
|
if os.path.islink(p) or os.path.isfile(p):
|
|
os.remove(p)
|
|
n += 1
|
|
for dp, _dns, _fns in os.walk(root(), topdown=False):
|
|
if dp != root():
|
|
try:
|
|
os.rmdir(dp)
|
|
except OSError:
|
|
pass
|
|
return n
|
|
|
|
|
|
def import_record(target):
|
|
"""What gets stored on the image after a verified import."""
|
|
m, _ = load_manifest()
|
|
s = load_state() or {}
|
|
return {"upload_id": m["upload_id"] if m else None, "imported": _now(), "target": target,
|
|
"source_pc": (m or {}).get("source_pc"), "mcl": (m or {}).get("mcl"),
|
|
"os": s.get("os"), "files": len(m["files"]) if m else 0}
|
|
|
|
|
|
def finish_import(image_root, target):
|
|
"""After the import moved the files: keep a record on the image, then
|
|
clear the manifest and state so the folder starts fresh. Call with
|
|
the lock held."""
|
|
rec = import_record(target)
|
|
try:
|
|
_write_json(os.path.join(image_root, ".mcl-import.json"), rec)
|
|
except OSError:
|
|
pass
|
|
for name in TOP_FILES:
|
|
try:
|
|
os.remove(os.path.join(root(), name))
|
|
except OSError:
|
|
pass
|
|
return rec
|
|
|
|
|
|
def last_import(image_root):
|
|
try:
|
|
return _read_json(os.path.join(image_root, ".mcl-import.json"))
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def try_lock():
|
|
"""For the import route: take the lock without waiting (True if held)."""
|
|
return _lock.acquire(blocking=False)
|
|
|
|
|
|
def unlock():
|
|
_lock.release()
|