1467 lines
58 KiB
Python
1467 lines
58 KiB
Python
"""Bring the vendor BPRT folder from the work PC into _shared/Applications/BPRT.
|
|
|
|
BPRT (every image's Deploy/Applications/BPRT links to it; PESetup copies it to
|
|
every PC while it images) is a vendor folder of about 7.7 GB: the six MCL
|
|
packages, the Office zips and the bulk-enroll orchestrator scripts. The vendor
|
|
keeps it current on the user's GE work PC with MediaCreator Lite; this server
|
|
is only reachable from there over the GE network (this webapp and SSH, no
|
|
SMB), and never reaches out itself. So the user runs
|
|
webapp/static/Sync-BPRT.ps1 there (downloadable from the BPRT page), which:
|
|
|
|
1. GETs manifest.json (path, size, sha256 of every vendor file here);
|
|
2. hashes its own copy and uploads only new or changed files, plus a
|
|
sync.json listing the complete intended vendor folder, with one sftp
|
|
batch into BPRT-incoming/<batch>/.
|
|
|
|
This module then, each step a background job (python3 -m services.bprt_sync):
|
|
|
|
* manifest hashes the live folder (cached by path, size and mtime in
|
|
UPDATES_DIR/bprt/hashcache.json; only the first run reads all
|
|
7.7 GB);
|
|
* validate checks an incoming batch: every listed file uploaded with the
|
|
right hash or unchanged here, the vendor's one-package-per-prefix
|
|
rule (services/mcl_ppkg.py), the scripts the orchestrator calls;
|
|
and works out the diff;
|
|
* activate builds BPRT.new from hard links (live files that did not change,
|
|
plus the incoming ones), applies the site overlay, and swaps it
|
|
in with two renames: BPRT to BPRT-archive/full-<ts>, BPRT.new to
|
|
BPRT. Rollback swaps an archive back the same way.
|
|
* unzip the fallback when SSH is not available: a zip uploaded on the
|
|
page is unpacked into a new incoming batch, then validated.
|
|
|
|
Hard links: an archive shares every unchanged file with the live folder, so
|
|
it costs only the files that changed. Nothing here edits a file in place;
|
|
replace files, never rewrite them, or the archives change with them.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import errno
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import stat
|
|
import sys
|
|
import time
|
|
import zipfile
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
|
|
from services import mcl_ppkg
|
|
from services import updates_common as uc
|
|
|
|
UNIT = "pxe-bprt-sync"
|
|
|
|
# The site overlay: files in the live BPRT that are ours, not the vendor's.
|
|
# manifest.json leaves them out (the work PC never sees or replaces them) and
|
|
# activation carries them into every new BPRT from the live one.
|
|
# Select-PCConfig.ps1 our shim, repo playbook/shopfloor-setup/BPRT/
|
|
# Select-PCConfig.ps1: returns the tenant and
|
|
# purpose startnet.cmd pre-seeded, else runs the
|
|
# vendor dialog, which is kept beside it as
|
|
# Select-PCConfig-vendor.ps1 (VENDOR_RENAMED);
|
|
# Register-BulkEnrollTask-Standard.ps1 registers the orchestrator; specialize Order 39
|
|
# of playbook/FlatUnattendW10.xml and
|
|
# FlatUnattendW10-engineer.xml runs it. It exists
|
|
# only on the share (no repo copy);
|
|
# Register-BulkEnrollTask.ps1 the shopfloor -ManualFallback variant. Nothing
|
|
# calls it since the shopfloor unattend dropped
|
|
# its Order 16 (2026-08-06); kept as it was.
|
|
OVERLAY = ("Select-PCConfig.ps1", "Register-BulkEnrollTask-Standard.ps1",
|
|
"Register-BulkEnrollTask.ps1")
|
|
# Vendor files the overlay displaces, and the name the vendor's copy keeps.
|
|
VENDOR_RENAMED = {"Select-PCConfig.ps1": "Select-PCConfig-vendor.ps1"}
|
|
# Our shim calls the vendor copy by this name; the vendor dialog never does.
|
|
SHIM_MARKER = b"Select-PCConfig-vendor.ps1"
|
|
# Overlay files enrollment cannot work without (the unattends call it).
|
|
OVERLAY_REQUIRED = ("Register-BulkEnrollTask-Standard.ps1",)
|
|
|
|
# Vendor files the bulk-enroll flow calls by path (C:\Deploy\Applications\BPRT\...):
|
|
# a drop without one of them breaks enrollment on every PC. Read from
|
|
# Start-BulkEnrollOrchestrator.ps1 and the scripts it starts, plus the two the
|
|
# unattends reach. Translations\*.psd1 are optional (WCD-Helpers.ps1 falls
|
|
# back to English).
|
|
REQUIRED = (
|
|
"Start-BulkEnrollOrchestrator.cmd", # what Register-BulkEnrollTask-Standard.ps1 registers
|
|
"Start-BulkEnrollOrchestrator.ps1",
|
|
"WCD-Helpers.ps1", # dot-sourced by the orchestrator and every dialog
|
|
"Show-ImageUntilNetworkConnected.ps1",
|
|
"Waiting For Internet Connection.png",
|
|
"Select-PCConfig.ps1", # the vendor dialog, kept as Select-PCConfig-vendor.ps1
|
|
"GE Aerospace Transparent Logo.png",
|
|
"Show-PleaseWait.ps1",
|
|
"Translations.json",
|
|
"ForegroundWindowWithProgress.exe",
|
|
"Insert-BPRTToPPKG.ps1",
|
|
"Send-OEMSkipPA.ps1",
|
|
"Wait-IntuneEnrollmentAndReboot.ps1",
|
|
"Show-VanillaConfirmation.ps1", # the unattends' 'vanilla' Shift+F10 alias
|
|
)
|
|
|
|
# Windows PowerShell 5.1 reads a script without a BOM as ANSI, which garbles
|
|
# any non-ASCII text in it; the vendor ships five such scripts. Activation
|
|
# gives those a UTF-8 BOM (what was done by hand for the v2.19 drop), and the
|
|
# manifest also reports their hash without it so the work PC does not see
|
|
# them as changed.
|
|
BOM = b"\xef\xbb\xbf"
|
|
SCRIPT_EXTS = (".ps1", ".psm1", ".psd1")
|
|
|
|
_BATCH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$")
|
|
_ARCHIVE = re.compile(r"^full-\d{8}-\d{6}(-\d+)?$")
|
|
_WIN_BAD = re.compile(r'[<>:"|?*\\\x00-\x1f]')
|
|
_DEFAULT_FILE_MODE = 0o744 # what the share's BPRT files carry (see mcl_ppkg)
|
|
|
|
|
|
# --- paths -------------------------------------------------------------------
|
|
|
|
def bprt_dir():
|
|
return mcl_ppkg.bprt_dir()
|
|
|
|
|
|
def new_dir():
|
|
return bprt_dir() + ".new"
|
|
|
|
|
|
def archive_root():
|
|
return mcl_ppkg.archive_dir()
|
|
|
|
|
|
def incoming_root():
|
|
return os.path.join(os.path.dirname(bprt_dir()), "BPRT-incoming")
|
|
|
|
|
|
def uploads_dir():
|
|
return os.path.join(incoming_root(), ".uploads")
|
|
|
|
|
|
def _state(*parts):
|
|
return uc.state_dir("bprt", *parts)
|
|
|
|
|
|
def _path(name):
|
|
return os.path.join(_state(), name)
|
|
|
|
|
|
def _stamp():
|
|
return datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
|
|
|
|
def check_batch(name):
|
|
name = str(name or "")
|
|
if not _BATCH.match(name) or name.endswith(".part"):
|
|
raise ValueError(f"{name or '(blank)'} is not an incoming batch name")
|
|
return name
|
|
|
|
|
|
def batch_dir(name):
|
|
return os.path.join(incoming_root(), check_batch(name))
|
|
|
|
|
|
def _owner():
|
|
"""(uid, gid, mode) of the live BPRT folder, else of its parent."""
|
|
for p in (bprt_dir(), os.path.dirname(bprt_dir())):
|
|
try:
|
|
st = os.stat(p)
|
|
return st.st_uid, st.st_gid, stat.S_IMODE(st.st_mode)
|
|
except OSError:
|
|
continue
|
|
return None, None, 0o755
|
|
|
|
|
|
def _match_owner(path, mode):
|
|
uid, gid, _ = _owner()
|
|
try:
|
|
if uid is not None:
|
|
os.chown(path, uid, gid)
|
|
os.chmod(path, mode)
|
|
except OSError:
|
|
pass # not root (local tests): the files are ours already
|
|
|
|
|
|
def ensure_incoming():
|
|
"""BPRT-incoming, writable by the share owner (the SSH user uploads as pxe)."""
|
|
path = incoming_root()
|
|
if not os.path.isdir(path):
|
|
os.makedirs(path, exist_ok=True)
|
|
_match_owner(path, _owner()[2] or 0o755)
|
|
return path
|
|
|
|
|
|
# --- walking and hashing -----------------------------------------------------
|
|
|
|
def walk(root):
|
|
"""({relpath: stat} of the regular files under root, [skipped relpaths]).
|
|
Relpaths use forward slashes; symlinks and special files are skipped."""
|
|
files, skipped = {}, []
|
|
if not os.path.isdir(root):
|
|
return files, skipped
|
|
for here, dirs, names in os.walk(root):
|
|
dirs.sort()
|
|
rel_here = os.path.relpath(here, root)
|
|
for d in list(dirs):
|
|
if os.path.islink(os.path.join(here, d)):
|
|
dirs.remove(d)
|
|
skipped.append(_rel(rel_here, d) + "/")
|
|
for n in sorted(names):
|
|
rel = _rel(rel_here, n)
|
|
st = os.lstat(os.path.join(here, n))
|
|
if stat.S_ISREG(st.st_mode):
|
|
files[rel] = st
|
|
else:
|
|
skipped.append(rel)
|
|
return files, skipped
|
|
|
|
|
|
def _rel(rel_here, name):
|
|
return name if rel_here == "." else f"{rel_here.replace(os.sep, '/')}/{name}"
|
|
|
|
|
|
def _is_script(rel):
|
|
return rel.lower().endswith(SCRIPT_EXTS)
|
|
|
|
|
|
def hash_file(path, rel=None, progress=None):
|
|
"""(sha256, sha256 of the content after a UTF-8 BOM or None), uppercase,
|
|
in one read. The second is only computed for scripts that start with a
|
|
BOM."""
|
|
h, h2 = hashlib.sha256(), None
|
|
script = _is_script(rel or path)
|
|
with open(path, "rb") as f:
|
|
first = True
|
|
for block in iter(lambda: f.read(1 << 22), b""):
|
|
h.update(block)
|
|
if first:
|
|
first = False
|
|
if script and block.startswith(BOM):
|
|
h2 = hashlib.sha256(block[len(BOM):])
|
|
elif h2 is not None:
|
|
h2.update(block)
|
|
if progress:
|
|
progress(len(block))
|
|
return h.hexdigest().upper(), (h2.hexdigest().upper() if h2 else None)
|
|
|
|
|
|
def _key(rel, st):
|
|
return f"{rel}|{st.st_size}|{st.st_mtime_ns}"
|
|
|
|
|
|
class HashCache:
|
|
"""sha256 per (relpath, size, mtime), shared by the live folder, the
|
|
archives and BPRT.new: a hard link keeps size and mtime, so a file that
|
|
only moved between them is never read again."""
|
|
|
|
def __init__(self):
|
|
data = uc.read_json(_path("hashcache.json"), {}) or {}
|
|
self.entries = data.get("entries") if isinstance(data.get("entries"), dict) else {}
|
|
self.dirty = False
|
|
|
|
def get(self, rel, st):
|
|
e = self.entries.get(_key(rel, st))
|
|
if e and time.time() - e.get("seen", 0) > 86400:
|
|
e["seen"] = int(time.time()) # still in use: keep it past the pruning age
|
|
self.dirty = True
|
|
return e
|
|
|
|
def put(self, rel, st, sha, nobom):
|
|
self.entries[_key(rel, st)] = {"sha256": sha, "nobom": nobom, "seen": int(time.time())}
|
|
self.dirty = True
|
|
|
|
def hash(self, root, rel, st, progress=None):
|
|
"""Cached entry for root/rel, hashing it on a miss. A file that
|
|
changed while it was read is returned but not cached."""
|
|
e = self.get(rel, st)
|
|
if e:
|
|
return e
|
|
path = os.path.join(root, *rel.split("/"))
|
|
sha, nobom = hash_file(path, rel, progress)
|
|
after = os.lstat(path)
|
|
e = {"sha256": sha, "nobom": nobom}
|
|
if (after.st_size, after.st_mtime_ns) == (st.st_size, st.st_mtime_ns):
|
|
self.put(rel, st, sha, nobom)
|
|
return e
|
|
|
|
def save(self, max_age_days=180):
|
|
if not self.dirty:
|
|
return
|
|
cutoff = time.time() - max_age_days * 86400
|
|
self.entries = {k: v for k, v in self.entries.items() if v.get("seen", 0) >= cutoff}
|
|
uc.write_json(_path("hashcache.json"), {"entries": self.entries})
|
|
self.dirty = False
|
|
|
|
|
|
def _same(entry, sha):
|
|
"""True when a file with this cache entry is the vendor file with this
|
|
sha256 (with or without the BOM we add)."""
|
|
sha = str(sha or "").upper()
|
|
return bool(sha) and sha in (entry.get("sha256"), entry.get("nobom"))
|
|
|
|
|
|
# --- the live folder ---------------------------------------------------------
|
|
|
|
def _lc(names):
|
|
return {n.lower() for n in names}
|
|
|
|
|
|
def is_shim(path):
|
|
try:
|
|
with open(path, "rb") as f:
|
|
return SHIM_MARKER.lower() in f.read(1 << 20).lower()
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def vendor_view(root, files):
|
|
"""{vendor path: relpath in root} for a BPRT folder: the overlay left
|
|
out, Select-PCConfig-vendor.ps1 under the vendor's own name. A folder the
|
|
overlay was never applied to still has the vendor's Select-PCConfig.ps1."""
|
|
overlay = _lc(OVERLAY)
|
|
back = {v.lower(): k for k, v in VENDOR_RENAMED.items()}
|
|
out = {}
|
|
for rel in files:
|
|
low = rel.lower()
|
|
if low in back:
|
|
out[back[low]] = rel
|
|
elif low not in overlay:
|
|
out[rel] = rel
|
|
for vendor, kept in VENDOR_RENAMED.items():
|
|
if vendor not in out and vendor in files and not is_shim(os.path.join(root, vendor)):
|
|
out[vendor] = vendor
|
|
return out
|
|
|
|
|
|
def overlay_sources(root, files):
|
|
"""{overlay name: absolute source path} found in a BPRT folder; our shim
|
|
falls back to the repo copy when the webapp runs from a checkout."""
|
|
out = {}
|
|
for name in OVERLAY:
|
|
path = os.path.join(root, name)
|
|
if name in files and (name not in VENDOR_RENAMED or is_shim(path)):
|
|
out[name] = path
|
|
if "Select-PCConfig.ps1" not in out:
|
|
here = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
repo = os.path.join(here, "..", "playbook", "shopfloor-setup", "BPRT", "Select-PCConfig.ps1")
|
|
if is_shim(repo):
|
|
out["Select-PCConfig.ps1"] = os.path.abspath(repo)
|
|
return out
|
|
|
|
|
|
def manifest(build=False, progress=None):
|
|
"""What manifest.json serves: every vendor file of the live BPRT with size
|
|
and sha256 (and sha256_nobom where we added a BOM). Returns None when a
|
|
file is not hashed yet and build is False (the caller starts the job)."""
|
|
root = bprt_dir()
|
|
if os.path.islink(root) or not os.path.isdir(root):
|
|
raise RuntimeError(f"{root} is missing or not a folder")
|
|
files, _ = walk(root)
|
|
view = vendor_view(root, files)
|
|
cache = HashCache()
|
|
missing = [rel for rel in view.values() if not cache.get(rel, files[rel])]
|
|
if missing and not build:
|
|
return None
|
|
if progress:
|
|
progress.start(sum(files[r].st_size for r in missing), len(missing))
|
|
out = []
|
|
for vpath in sorted(view, key=str.lower):
|
|
rel = view[vpath]
|
|
st = files[rel]
|
|
if progress:
|
|
progress.current(vpath)
|
|
e = cache.hash(root, rel, st, progress.add if progress else None)
|
|
row = {"path": vpath, "size": st.st_size, "sha256": e["sha256"]}
|
|
if e.get("nobom"):
|
|
row["sha256_nobom"] = e["nobom"]
|
|
if rel != vpath:
|
|
row["live_name"] = rel
|
|
out.append(row)
|
|
cache.save()
|
|
return {
|
|
"format": 1,
|
|
"generated": uc.now(),
|
|
"bprt": root,
|
|
"incoming": incoming_root(),
|
|
# Never uploaded: the site files the vendor does not also ship.
|
|
"overlay": [n for n in OVERLAY if n not in VENDOR_RENAMED],
|
|
"renamed": dict(VENDOR_RENAMED),
|
|
"count": len(out),
|
|
"bytes": sum(r["size"] for r in out),
|
|
"files": out,
|
|
}
|
|
|
|
|
|
def _bom_less_non_ascii(path):
|
|
"""True for a script PowerShell 5.1 would misread: valid UTF-8 with
|
|
non-ASCII text and no BOM."""
|
|
try:
|
|
with open(path, "rb") as f:
|
|
data = f.read()
|
|
except OSError:
|
|
return False
|
|
if data.startswith(BOM) or data.isascii():
|
|
return False
|
|
try:
|
|
data.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return False # ANSI already: PowerShell reads it as written
|
|
return True
|
|
|
|
|
|
def live_summary():
|
|
"""Facts for the page about the live BPRT (no hashing)."""
|
|
root = bprt_dir()
|
|
out = {"path": root, "exists": os.path.isdir(root), "is_link": os.path.islink(root),
|
|
"count": 0, "bytes": 0, "newest": None, "overlay": [], "missing_required": [],
|
|
"bom_needed": [], "skipped": []}
|
|
if not out["exists"]:
|
|
return out
|
|
files, out["skipped"] = walk(root)
|
|
view = vendor_view(root, files)
|
|
out["count"] = len(files)
|
|
out["bytes"] = sum(st.st_size for st in files.values())
|
|
out["newest"] = max((st.st_mtime for st in files.values()), default=None)
|
|
srcs = overlay_sources(root, files)
|
|
for name in OVERLAY:
|
|
row = {"name": name, "present": name in files, "ok": name in srcs and srcs[name].startswith(root)}
|
|
if name in VENDOR_RENAMED:
|
|
row["vendor_copy"] = VENDOR_RENAMED[name] in files
|
|
row["is_shim"] = row["ok"]
|
|
out["overlay"].append(row)
|
|
out["missing_required"] = [r for r in REQUIRED if r not in view]
|
|
out["bom_needed"] = [rel for rel in files if _is_script(rel)
|
|
and _bom_less_non_ascii(os.path.join(root, *rel.split("/")))]
|
|
return out
|
|
|
|
|
|
# --- job state ---------------------------------------------------------------
|
|
|
|
def job_state():
|
|
state = uc.read_json(_path("job.json"), {}) or {}
|
|
running = bool(state.get("state") == "running" and uc.job_running(UNIT))
|
|
if state.get("state") == "running" and not running:
|
|
state["state"] = "interrupted"
|
|
state["running"] = running
|
|
total = state.get("total_bytes") or 0
|
|
done = state.get("done_bytes") or 0
|
|
state["percent"] = min(100.0, round(100.0 * done / total, 1)) if total else None
|
|
return state
|
|
|
|
|
|
def _job_update(**kw):
|
|
state = uc.read_json(_path("job.json"), {}) or {}
|
|
state.update(kw)
|
|
uc.write_json(_path("job.json"), state)
|
|
|
|
|
|
class Progress:
|
|
"""Byte progress of a job, written to job.json at most once a second."""
|
|
|
|
def __init__(self, step=""):
|
|
self.done = 0
|
|
self.last = 0.0
|
|
if step:
|
|
_job_update(step=step)
|
|
|
|
def start(self, total, count=0):
|
|
self.done = 0
|
|
_job_update(total_bytes=total, done_bytes=0, files=count)
|
|
|
|
def current(self, name):
|
|
self.name = name
|
|
|
|
def add(self, n):
|
|
self.done += n
|
|
if time.time() - self.last >= 1:
|
|
self.last = time.time()
|
|
_job_update(done_bytes=self.done, current=getattr(self, "name", ""))
|
|
|
|
def step(self, text):
|
|
_job_update(step=text, current="")
|
|
|
|
|
|
def busy():
|
|
"""Kind of the running job, or ''."""
|
|
st = job_state()
|
|
return st.get("kind", "") if st.get("running") else ""
|
|
|
|
|
|
def start(kind, arg=""):
|
|
"""Start a background job; one at a time (they all read or swap BPRT)."""
|
|
if kind not in ("manifest", "validate", "activate", "unzip"):
|
|
raise ValueError(f"unknown job {kind}")
|
|
if busy() or uc.job_running(UNIT):
|
|
raise RuntimeError(f"a BPRT job ({busy() or 'unknown'}) is still running")
|
|
if kind in ("validate", "activate"):
|
|
check_batch(arg)
|
|
if kind == "unzip":
|
|
clean_zip_name(arg) # also keeps a name like --help out of the job's argv
|
|
if kind == "activate":
|
|
res = validation(arg)
|
|
if not res or res.get("state") != "valid":
|
|
raise RuntimeError("validate this batch first; only a valid batch can be activated")
|
|
uc.write_json(_path("job.json"), {"kind": kind, "arg": arg, "state": "running",
|
|
"started": uc.now(), "step": "Starting"})
|
|
try:
|
|
uc.start_job(UNIT, ["services.bprt_sync", kind] + ([arg] if arg else []))
|
|
except Exception as ex:
|
|
if not uc.job_running(UNIT): # lost a race to another start: leave its state alone
|
|
_job_update(state="error", finished=uc.now(), error=str(ex)[:400])
|
|
raise
|
|
|
|
|
|
def run_job(kind, arg=""):
|
|
"""Body of a job (also called directly by tests): runs it and records the
|
|
outcome in job.json."""
|
|
_job_update(kind=kind, arg=arg, state="running", error="", result="",
|
|
total_bytes=0, done_bytes=0, current="")
|
|
try:
|
|
if kind == "manifest":
|
|
m = manifest(build=True, progress=Progress("Hashing the live BPRT"))
|
|
result = f"{m['count']} files hashed"
|
|
elif kind == "validate":
|
|
res = validate(arg, Progress("Validating"))
|
|
result = res["state"]
|
|
elif kind == "activate":
|
|
res = activate(arg, Progress("Building BPRT.new"))
|
|
result = f"activated, previous BPRT in {res['archive']}"
|
|
elif kind == "unzip":
|
|
batch = unzip(arg, Progress("Unpacking"))
|
|
res = validate(batch, Progress("Validating"))
|
|
result = f"unpacked into {batch}: {res['state']}"
|
|
else:
|
|
raise ValueError(f"unknown job {kind}")
|
|
except Exception as ex:
|
|
_job_update(state="error", finished=uc.now(), error=str(ex)[:600], step="")
|
|
raise
|
|
st = uc.read_json(_path("job.json"), {}) or {}
|
|
_job_update(state="done", finished=uc.now(), result=result, step="", current="",
|
|
done_bytes=st.get("total_bytes") or 0)
|
|
return result
|
|
|
|
|
|
class _Busy(RuntimeError):
|
|
pass
|
|
|
|
|
|
class _Lock:
|
|
"""Exclusive lock around anything that builds or swaps BPRT, shared by the
|
|
jobs, the inline rollback and recover()."""
|
|
|
|
def __enter__(self):
|
|
self.fd = os.open(_path(".lock"), os.O_CREAT | os.O_RDWR, 0o644)
|
|
try:
|
|
fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
os.close(self.fd)
|
|
raise _Busy("another BPRT activation or rollback is running") from None
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
fcntl.flock(self.fd, fcntl.LOCK_UN)
|
|
os.close(self.fd)
|
|
return False
|
|
|
|
|
|
# --- incoming batches --------------------------------------------------------
|
|
|
|
def _read_sync(folder):
|
|
path = os.path.join(folder, "sync.json")
|
|
try:
|
|
with open(path, encoding="utf-8-sig") as f:
|
|
data = json.load(f)
|
|
except OSError:
|
|
return None
|
|
except ValueError as ex:
|
|
raise ValueError(f"sync.json is not valid JSON: {ex}") from None
|
|
if not isinstance(data, dict):
|
|
raise ValueError("sync.json is not a JSON object")
|
|
return data
|
|
|
|
|
|
def _list(value):
|
|
"""A JSON list from sync.json. Windows PowerShell 5.1's ConvertTo-Json
|
|
writes an array that carries PowerShell's extra type data as
|
|
{"value": [...], "Count": n}; take the list out of that too."""
|
|
if isinstance(value, dict) and isinstance(value.get("value"), list):
|
|
value = value["value"]
|
|
return value if isinstance(value, list) else []
|
|
|
|
|
|
def validation(name):
|
|
return uc.read_json(os.path.join(_state("batches"), f"{check_batch(name)}.json"), None)
|
|
|
|
|
|
def batches():
|
|
"""Incoming batches, newest first: name, files, bytes, sync.json facts and
|
|
the last validation. A '<name>.part' folder is an sftp upload still
|
|
running (Sync-BPRT.ps1 renames it when done)."""
|
|
root = incoming_root()
|
|
out = []
|
|
if not os.path.isdir(root):
|
|
return out
|
|
live_files = None
|
|
for name in os.listdir(root):
|
|
folder = os.path.join(root, name)
|
|
if name.startswith(".") or not os.path.isdir(folder) or os.path.islink(folder):
|
|
continue
|
|
uploading = name.endswith(".part")
|
|
base = name[:-5] if uploading else name
|
|
if not _BATCH.match(base):
|
|
continue
|
|
files, _ = walk(folder)
|
|
row = {"name": name, "uploading": uploading,
|
|
"files": len([f for f in files if f != "sync.json"]),
|
|
"bytes": sum(st.st_size for f, st in files.items() if f != "sync.json"),
|
|
"modified": max([st.st_mtime for st in files.values()] or [os.stat(folder).st_mtime]),
|
|
"sync": None, "sync_error": "", "validation": None}
|
|
if not uploading:
|
|
try:
|
|
sync = _read_sync(folder)
|
|
if sync:
|
|
row["sync"] = {k: sync.get(k) for k in ("computer", "user", "created", "source")}
|
|
row["sync"]["listed"] = len(_list(sync.get("files")))
|
|
except ValueError as ex:
|
|
row["sync_error"] = str(ex)
|
|
res = validation(name)
|
|
if res:
|
|
if live_files is None:
|
|
live_files = walk(bprt_dir())[0]
|
|
res["stale"] = _stale(res, live_files)
|
|
row["validation"] = res
|
|
out.append(row)
|
|
out.sort(key=lambda r: r["modified"], reverse=True)
|
|
return out
|
|
|
|
|
|
def _snapshot(files):
|
|
"""{relpath: [size, mtime_ns]} of a walk(), as a validation records it."""
|
|
return {rel: [st.st_size, st.st_mtime_ns] for rel, st in files.items()}
|
|
|
|
|
|
def _live_moved(saved, files):
|
|
"""What differs between a recorded snapshot of the live BPRT and its
|
|
files now, or ''. Any file added, removed or changed there (an MCL upload
|
|
on /enrollment, an edit over SMB) means the validated plan, which decides
|
|
what the new folder keeps, is out of date."""
|
|
for rel in sorted(set(saved) | set(files), key=str.lower):
|
|
if rel not in files:
|
|
return f"{rel} is gone from the live BPRT"
|
|
if rel not in saved:
|
|
return f"{rel} was added to the live BPRT"
|
|
st = files[rel]
|
|
if [st.st_size, st.st_mtime_ns] != list(saved[rel]):
|
|
return f"{rel} in the live BPRT changed"
|
|
return ""
|
|
|
|
|
|
def _stale(res, live_files=None):
|
|
"""Why a validation no longer holds (a file changed since), or ''.
|
|
live_files: a walk() of the live BPRT, when the caller has one."""
|
|
try:
|
|
folder = batch_dir(res["batch"])
|
|
except ValueError:
|
|
return "bad batch name"
|
|
if res.get("live_all") is not None:
|
|
if live_files is None:
|
|
live_files = walk(bprt_dir())[0]
|
|
why = _live_moved(res["live_all"], live_files)
|
|
if why:
|
|
return f"{why} since the batch was validated"
|
|
for path, (size, mtime) in (res.get("batch_stat") or {}).items():
|
|
try:
|
|
st = os.lstat(os.path.join(folder, *path.split("/")))
|
|
except OSError:
|
|
return f"{path} is gone from the batch"
|
|
if (st.st_size, st.st_mtime_ns) != (size, mtime):
|
|
return f"{path} in the batch changed since it was validated"
|
|
for rel, (size, mtime) in (res.get("live_stat") or {}).items():
|
|
try:
|
|
st = os.lstat(os.path.join(bprt_dir(), *rel.split("/")))
|
|
except OSError:
|
|
return f"{rel} is gone from the live BPRT"
|
|
if (st.st_size, st.st_mtime_ns) != (size, mtime):
|
|
return f"{rel} in the live BPRT changed since the batch was validated"
|
|
return ""
|
|
|
|
|
|
def _check_rel(path):
|
|
"""A relative path from sync.json, or ValueError: forward slashes, no
|
|
empty, '.' or '..' parts, nothing Windows cannot name."""
|
|
if not isinstance(path, str) or not path or path.startswith("/") or len(path) > 240:
|
|
raise ValueError(f"bad path {path!r}")
|
|
parts = path.split("/")
|
|
for p in parts:
|
|
if p in ("", ".", "..") or _WIN_BAD.search(p) or p != p.rstrip(" ."):
|
|
raise ValueError(f"bad path {path!r}")
|
|
return path
|
|
|
|
|
|
def _dest(vpath):
|
|
"""Name a vendor file gets in BPRT. Case-insensitive, as on the PCs: a
|
|
vendor select-pcconfig.ps1 next to our Select-PCConfig.ps1 would be one
|
|
file to Windows."""
|
|
for vendor, kept in VENDOR_RENAMED.items():
|
|
if vpath.lower() == vendor.lower():
|
|
return kept
|
|
return vpath
|
|
|
|
|
|
def _ppkg_names(names):
|
|
"""Top-level names the enroll scripts may treat as packages."""
|
|
return [n for n in names if "/" not in n and (n.lower().endswith(".ppkg") or mcl_ppkg.is_mcl_name(n))]
|
|
|
|
|
|
def _versions(names):
|
|
"""{prefix: file name} for the vendor-valid MCL packages among names."""
|
|
out = {}
|
|
for n in names:
|
|
if mcl_ppkg.is_mcl_name(n):
|
|
out.setdefault(mcl_ppkg.parse(n, strict=False)["prefix"], []).append(n)
|
|
return out
|
|
|
|
|
|
def check_packages(names, errors, warnings):
|
|
"""The vendor rules on the package names a folder will hold."""
|
|
pk = _ppkg_names(names)
|
|
mcl = [n for n in pk if mcl_ppkg.is_mcl_name(n)]
|
|
for n in pk:
|
|
if n not in mcl:
|
|
warnings.append(f"{n} does not match the vendor's package naming; the enroll scripts ignore it")
|
|
elif "_exp_xxxxxxxx_" not in n.lower():
|
|
warnings.append(f"{n} has no Exp_XXXXXXXX placeholder: a package already used on a PC, "
|
|
"the enroll script cannot inject a new token into it")
|
|
if not mcl:
|
|
errors.append("no MCL package (*_v<major>.<minor>.ppkg) in the new folder")
|
|
seen = set()
|
|
for n, others in sorted(mcl_ppkg._clashes(mcl).items()):
|
|
prefix = mcl_ppkg.parse(n, strict=False)["prefix"]
|
|
group = frozenset([n, *others])
|
|
if group in seen:
|
|
continue # the same pair, seen from its other package
|
|
seen.add(group)
|
|
errors.append(f"{prefix} would match {n} and {', '.join(others)}: the enroll scripts fail "
|
|
"when one tenant/environment/purpose prefix finds two packages")
|
|
return mcl
|
|
|
|
|
|
def validate(name, progress=None):
|
|
"""Check an incoming batch and work out what activating it would change.
|
|
Saves and returns the result (state 'valid' or 'invalid')."""
|
|
folder = batch_dir(name)
|
|
progress = progress or Progress()
|
|
res = {"batch": name, "validated": uc.now(), "state": "invalid", "errors": [],
|
|
"warnings": [], "sync": {}, "plan": [], "batch_stat": {}, "live_stat": {}}
|
|
errors, warnings = res["errors"], res["warnings"]
|
|
try:
|
|
_validate(folder, res, progress)
|
|
except ValueError as ex:
|
|
errors.append(str(ex))
|
|
if not errors:
|
|
res["state"] = "valid"
|
|
uc.write_json(os.path.join(_state("batches"), f"{name}.json"), res)
|
|
return res
|
|
|
|
|
|
def _validate(folder, res, progress):
|
|
errors, warnings = res["errors"], res["warnings"]
|
|
if not os.path.isdir(folder):
|
|
raise ValueError("the batch folder is gone")
|
|
sync = _read_sync(folder)
|
|
if sync is None:
|
|
raise ValueError("no sync.json in the batch: Sync-BPRT.ps1 uploads it last, so the upload "
|
|
"did not finish (or this folder was made by hand)")
|
|
res["sync"] = {k: sync.get(k) for k in ("computer", "user", "created", "source", "tool")}
|
|
listed = _list(sync.get("files"))
|
|
if not isinstance(listed, list) or not listed:
|
|
raise ValueError("sync.json lists no files")
|
|
want, seen_lc = {}, {}
|
|
for row in listed:
|
|
if not isinstance(row, dict):
|
|
raise ValueError("sync.json: every file entry must be an object")
|
|
path = _check_rel(row.get("path"))
|
|
if path.lower() in seen_lc:
|
|
raise ValueError(f"sync.json lists {path} twice (names differing only by case "
|
|
"are one file on Windows)")
|
|
seen_lc[path.lower()] = path
|
|
try:
|
|
size = int(row.get("size"))
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"sync.json: {path} has no size") from None
|
|
sha = str(row.get("sha256") or "").upper()
|
|
if not re.fullmatch(r"[0-9A-F]{64}", sha):
|
|
raise ValueError(f"sync.json: {path} has no sha256")
|
|
want[path] = {"size": size, "sha256": sha}
|
|
for e in _list(sync.get("parse_errors")):
|
|
if isinstance(e, dict):
|
|
errors.append(f"{e.get('path')}: PowerShell could not parse it on the work PC "
|
|
f"({str(e.get('error', ''))[:200]}); the MediaCreator cache may be corrupt")
|
|
claimed = set(x for x in _list(sync.get("uploaded")) if isinstance(x, str))
|
|
|
|
have, _ = walk(folder)
|
|
have.pop("sync.json", None)
|
|
root = bprt_dir()
|
|
if os.path.islink(root) or not os.path.isdir(root):
|
|
raise ValueError(f"{root} is missing or not a folder")
|
|
live, skipped = walk(root)
|
|
res["live_all"] = _snapshot(live)
|
|
view = vendor_view(root, live)
|
|
# Windows names are case-insensitive: the work PC sees a file the vendor
|
|
# renamed by case only as unchanged and does not upload it, so a listed
|
|
# path finds its live file case-insensitively (exact spelling first).
|
|
view_lc = {vp.lower(): vp for vp in sorted(view, reverse=True)}
|
|
want_lc = _lc(want)
|
|
cache = HashCache()
|
|
|
|
def live_match(path):
|
|
return path if path in view else view_lc.get(path.lower())
|
|
|
|
# Hash what was uploaded and what in the live folder is not cached yet.
|
|
todo_live = [rel for vp, rel in view.items() if vp.lower() in want_lc and not cache.get(rel, live[rel])]
|
|
progress.start(sum(st.st_size for st in have.values())
|
|
+ sum(live[r].st_size for r in todo_live), len(have) + len(todo_live))
|
|
got = {}
|
|
for path in sorted(have):
|
|
progress.current(path)
|
|
st = have[path]
|
|
sha, nobom = hash_file(os.path.join(folder, *path.split("/")), path, progress.add)
|
|
got[path] = sha
|
|
res["batch_stat"][path] = [st.st_size, st.st_mtime_ns]
|
|
if path in want and sha == want[path]["sha256"]:
|
|
# Pre-seed the cache under the name it will have in BPRT.
|
|
cache.put(_dest(path), st, sha, nobom)
|
|
for path in sorted(set(have) - set(want)):
|
|
warnings.append(f"{path} was uploaded but is not in sync.json; it is ignored")
|
|
for path in sorted(claimed - set(have)):
|
|
if path in want:
|
|
errors.append(f"{path}: sync.json says it was uploaded but it is not in the batch")
|
|
|
|
plan, diff = [], {"new": [], "changed": [], "unchanged": [], "removed": []}
|
|
matched = set()
|
|
for path in sorted(want, key=str.lower):
|
|
w = want[path]
|
|
dest = _dest(path)
|
|
vp = live_match(path)
|
|
if vp is not None:
|
|
matched.add(vp)
|
|
if vp != path:
|
|
warnings.append(f"{vp} is now spelled {path} (a change of case only); the new BPRT "
|
|
"uses the new spelling")
|
|
if path in have:
|
|
if have[path].st_size != w["size"] or got[path] != w["sha256"]:
|
|
errors.append(f"{path}: hash mismatch (uploaded {have[path].st_size} bytes, sha256 "
|
|
f"{got[path][:12]}..., sync.json says {w['size']} bytes, "
|
|
f"{w['sha256'][:12]}...); upload it again")
|
|
continue
|
|
item = {"path": path, "dest": dest, "source": "incoming", "size": w["size"]}
|
|
if _is_script(path) and _bom_less_non_ascii(os.path.join(folder, *path.split("/"))):
|
|
item["bom_fix"] = True
|
|
if dest == VENDOR_RENAMED.get("Select-PCConfig.ps1") and is_shim(
|
|
os.path.join(folder, *path.split("/"))):
|
|
errors.append(f"{path} in the batch is our shim, not the vendor's dialog: kept as "
|
|
f"{dest} it would call itself. Upload the vendor folder, not the server's")
|
|
plan.append(item)
|
|
rel = view.get(vp) if vp else None
|
|
if rel is None:
|
|
diff["new"].append(path)
|
|
else:
|
|
progress.current(path)
|
|
e = cache.hash(root, rel, live[rel], progress.add)
|
|
res["live_stat"][rel] = [live[rel].st_size, live[rel].st_mtime_ns]
|
|
diff["unchanged" if _same(e, w["sha256"]) else "changed"].append(path)
|
|
else:
|
|
rel = view.get(vp) if vp else None
|
|
if rel is None:
|
|
errors.append(f"{path} was not uploaded and is not in the live BPRT")
|
|
continue
|
|
progress.current(path)
|
|
e = cache.hash(root, rel, live[rel], progress.add)
|
|
if not _same(e, w["sha256"]):
|
|
errors.append(f"{path} was not uploaded, but the live copy differs "
|
|
f"(live {e['sha256'][:12]}..., sync.json {w['sha256'][:12]}...); "
|
|
"run Sync-BPRT.ps1 again")
|
|
continue
|
|
res["live_stat"][rel] = [live[rel].st_size, live[rel].st_mtime_ns]
|
|
plan.append({"path": path, "dest": dest, "source": "live", "from": rel, "size": w["size"]})
|
|
diff["unchanged"].append(path)
|
|
diff["removed"] = sorted((vp for vp in view if vp not in matched), key=str.lower)
|
|
cache.save()
|
|
for s in skipped:
|
|
warnings.append(f"{s} in the live BPRT is a symlink or not a regular file; the new BPRT "
|
|
"will not have it (it stays in the archive)")
|
|
|
|
# The vendor rules and the files the flow calls by path.
|
|
check_packages(list(want), errors, warnings)
|
|
for path in sorted(have):
|
|
if path in want and path.lower().endswith(".ppkg"):
|
|
with open(os.path.join(folder, *path.split("/")), "rb") as f:
|
|
if f.read(len(mcl_ppkg.WIM_MAGIC)) != mcl_ppkg.WIM_MAGIC:
|
|
errors.append(f"{path} is not a provisioning package (no WIM header)")
|
|
for r in REQUIRED:
|
|
if r.lower() not in want_lc:
|
|
errors.append(f"{r} is missing: the bulk-enroll flow calls it by path")
|
|
for name in OVERLAY:
|
|
if name.lower() in _lc(want) and name not in VENDOR_RENAMED:
|
|
warnings.append(f"the vendor now ships {name}; the site copy replaces it as before")
|
|
for kept in VENDOR_RENAMED.values():
|
|
if kept.lower() in _lc(want):
|
|
errors.append(f"sync.json lists {kept}, the name the vendor's copy is kept under here")
|
|
srcs = overlay_sources(root, live)
|
|
if "Select-PCConfig.ps1" not in srcs:
|
|
errors.append("our Select-PCConfig.ps1 shim is not in the live BPRT (and no repo copy is "
|
|
"here): copy playbook/shopfloor-setup/BPRT/Select-PCConfig.ps1 there first")
|
|
for name in OVERLAY_REQUIRED:
|
|
if name not in srcs:
|
|
warnings.append(f"{name} is not in the live BPRT, so the new one will not have it "
|
|
"either: standard and engineer specialize (Order 39) calls it")
|
|
for path in diff["removed"]:
|
|
if path.lower().endswith(".zip"):
|
|
warnings.append(f"{path} is no longer shipped; MCL packages that install from it would fail")
|
|
|
|
old_pk = _versions(view)
|
|
new_pk = _versions(want)
|
|
ppkg = []
|
|
for prefix in sorted(set(old_pk) | set(new_pk), key=str.lower):
|
|
olds, news = old_pk.get(prefix, []), new_pk.get(prefix, [])
|
|
if sorted(olds) != sorted(news):
|
|
ppkg.append({"prefix": prefix,
|
|
"old": [{"name": n, "version": mcl_ppkg.parse(n, strict=False)["version"]} for n in olds],
|
|
"new": [{"name": n, "version": mcl_ppkg.parse(n, strict=False)["version"]} for n in news]})
|
|
zips = []
|
|
for path in diff["new"] + diff["changed"]:
|
|
if path.lower().endswith(".zip"):
|
|
vp = live_match(path)
|
|
rel = view[vp] if vp else None
|
|
zips.append({"path": path, "old_size": live[rel].st_size if rel else None,
|
|
"new_size": want[path]["size"]})
|
|
|
|
res["plan"] = plan
|
|
res["diff"] = diff
|
|
res["counts"] = {k: len(v) for k, v in diff.items()}
|
|
res["ppkg"] = ppkg
|
|
res["zips"] = zips
|
|
res["bom_fix"] = [i["path"] for i in plan if i.get("bom_fix")]
|
|
res["upload_bytes"] = sum(st.st_size for st in have.values())
|
|
res["total_bytes"] = sum(w["size"] for w in want.values())
|
|
|
|
|
|
# --- activation --------------------------------------------------------------
|
|
|
|
def _link(src, dst):
|
|
"""Hard link src at dst; a copy when the filesystem will not link."""
|
|
try:
|
|
os.link(src, dst)
|
|
except OSError as ex:
|
|
if ex.errno not in (errno.EXDEV, errno.EPERM, errno.EMLINK, errno.ENOTSUP, errno.EACCES):
|
|
raise
|
|
shutil.copy2(src, dst)
|
|
|
|
|
|
def _makedirs(root, rel_dir, live_root):
|
|
"""Create root/rel_dir level by level with the owner and mode of the same
|
|
folder in the live BPRT (else of BPRT itself)."""
|
|
if not rel_dir:
|
|
return
|
|
cur, live_cur = root, live_root
|
|
for part in rel_dir.split("/"):
|
|
cur = os.path.join(cur, part)
|
|
live_cur = os.path.join(live_cur, part)
|
|
if not os.path.isdir(cur):
|
|
os.mkdir(cur)
|
|
try:
|
|
mode = stat.S_IMODE(os.stat(live_cur).st_mode)
|
|
except OSError:
|
|
mode = _owner()[2] or 0o755
|
|
_match_owner(cur, mode)
|
|
|
|
|
|
def _file_mode(live_files, rel):
|
|
"""Mode for a new file: the one it replaces, else the most common one in
|
|
the live folder."""
|
|
st = live_files.get(rel)
|
|
if st:
|
|
return stat.S_IMODE(st.st_mode)
|
|
modes = Counter(stat.S_IMODE(s.st_mode) for s in live_files.values())
|
|
return modes.most_common(1)[0][0] if modes else _DEFAULT_FILE_MODE
|
|
|
|
|
|
def _unique(root, base):
|
|
path = os.path.join(root, base)
|
|
n = 1
|
|
while os.path.exists(path):
|
|
n += 1
|
|
path = os.path.join(root, f"{base}-{n}")
|
|
return path
|
|
|
|
|
|
def recover():
|
|
"""Finish or undo a swap a crash interrupted between its two renames, so
|
|
BPRT is never left missing. Returns what was done, or ''. Takes the lock:
|
|
while a job or rollback holds it, swap.json describes a swap in progress
|
|
(BPRT may be missing for a moment), not a crashed one, so it is left
|
|
alone; the holder cleans up itself."""
|
|
try:
|
|
with _Lock():
|
|
return _recover()
|
|
except _Busy:
|
|
return ""
|
|
|
|
|
|
def _recover():
|
|
"""recover() for a caller that already holds the lock."""
|
|
intent = uc.read_json(_path("swap.json"), None)
|
|
if not intent:
|
|
return ""
|
|
live = bprt_dir()
|
|
done = ""
|
|
if not os.path.exists(live):
|
|
if intent.get("new") and os.path.isdir(intent["new"]):
|
|
os.rename(intent["new"], live)
|
|
done = f"finished an interrupted swap: {os.path.basename(intent['new'])} is now BPRT"
|
|
elif intent.get("archive") and os.path.isdir(intent["archive"]):
|
|
os.rename(intent["archive"], live)
|
|
done = f"undid an interrupted swap: {os.path.basename(intent['archive'])} is BPRT again"
|
|
try:
|
|
os.remove(_path("swap.json"))
|
|
except OSError:
|
|
pass
|
|
if done:
|
|
_audit("BPRT_RECOVER", done)
|
|
return done
|
|
|
|
|
|
def _swap(src):
|
|
"""BPRT to BPRT-archive/full-<ts>, then src to BPRT. Returns the archive
|
|
path. SIGTERM (systemctl stop) waits until both renames are done."""
|
|
live = bprt_dir()
|
|
os.makedirs(archive_root(), exist_ok=True)
|
|
dest = _unique(archive_root(), f"full-{_stamp()}")
|
|
uc.write_json(_path("swap.json"), {"new": src, "archive": dest, "at": uc.now()})
|
|
old = None
|
|
try:
|
|
old = signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
except ValueError:
|
|
pass # not the main thread (the webapp's inline rollback)
|
|
try:
|
|
os.rename(live, dest)
|
|
try:
|
|
os.rename(src, live)
|
|
except OSError:
|
|
os.rename(dest, live)
|
|
raise
|
|
finally:
|
|
if old is not None:
|
|
signal.signal(signal.SIGTERM, old)
|
|
if os.path.isdir(live):
|
|
try:
|
|
os.remove(_path("swap.json"))
|
|
except OSError:
|
|
pass
|
|
# else both renames failed after the first: swap.json stays, so
|
|
# recover() puts BPRT back on the next page load or job.
|
|
return dest
|
|
|
|
|
|
def activate(name, progress=None):
|
|
"""Build BPRT.new from the validated batch and swap it in."""
|
|
progress = progress or Progress()
|
|
with _Lock():
|
|
_recover()
|
|
res = validation(name)
|
|
if not res or res.get("state") != "valid":
|
|
raise RuntimeError("this batch is not validated")
|
|
folder = batch_dir(name)
|
|
live = bprt_dir()
|
|
if os.path.islink(live) or not os.path.isdir(live):
|
|
raise RuntimeError(f"{live} is missing or not a folder")
|
|
live_files, _ = walk(live)
|
|
why = _stale(res, live_files)
|
|
if why:
|
|
raise RuntimeError(f"{why}; validate it again")
|
|
srcs = overlay_sources(live, live_files)
|
|
if "Select-PCConfig.ps1" not in srcs:
|
|
raise RuntimeError("our Select-PCConfig.ps1 shim is not in the live BPRT")
|
|
new = new_dir()
|
|
if os.path.lexists(new):
|
|
shutil.rmtree(new) # left by a failed run; never live
|
|
os.mkdir(new)
|
|
_match_owner(new, _owner()[2] or 0o755)
|
|
try:
|
|
progress.step("Building BPRT.new")
|
|
progress.start(len(res["plan"]))
|
|
for item in res["plan"]:
|
|
dest_rel = item["dest"]
|
|
dest = os.path.join(new, *dest_rel.split("/"))
|
|
_makedirs(new, dest_rel.rpartition("/")[0], live)
|
|
if item["source"] == "live":
|
|
_link(os.path.join(live, *item["from"].split("/")), dest)
|
|
else:
|
|
src = os.path.join(folder, *item["path"].split("/"))
|
|
if item.get("bom_fix"):
|
|
with open(src, "rb") as f:
|
|
data = f.read()
|
|
with open(dest, "wb") as f:
|
|
f.write(BOM + data)
|
|
shutil.copystat(src, dest)
|
|
else:
|
|
_link(src, dest)
|
|
_match_owner(dest, _file_mode(live_files, dest_rel))
|
|
if item["source"] == "live":
|
|
expect = res["live_stat"][item["from"]][0] # BOM-fixed earlier: 3 bytes more
|
|
else:
|
|
expect = item["size"] + (len(BOM) if item.get("bom_fix") else 0)
|
|
if os.path.getsize(dest) != expect:
|
|
raise RuntimeError(f"{dest_rel} has the wrong size in BPRT.new")
|
|
progress.add(1)
|
|
progress.step("Applying the site overlay")
|
|
top = os.listdir(new)
|
|
for oname, src in srcs.items():
|
|
dest = os.path.join(new, oname)
|
|
for n in top:
|
|
if n.lower() == oname.lower():
|
|
os.remove(os.path.join(new, n)) # the vendor shipped one (any case); ours wins
|
|
if src.startswith(live + os.sep):
|
|
_link(src, dest)
|
|
else:
|
|
shutil.copy2(src, dest) # the repo copy of our shim
|
|
_match_owner(dest, _file_mode(live_files, oname))
|
|
# Last look at the folder that is about to go live.
|
|
names, _ = walk(new)
|
|
errors = []
|
|
check_packages(list(names), errors, [])
|
|
missing = [r for r in REQUIRED if _dest(r).lower() not in _lc(names)]
|
|
if missing:
|
|
errors.append(f"missing {', '.join(missing)}")
|
|
if errors:
|
|
raise RuntimeError("BPRT.new failed its final check: " + "; ".join(errors))
|
|
# An MCL upload on /enrollment (or an SMB edit) while BPRT.new
|
|
# was built would be lost with the old folder: check again.
|
|
why = _live_moved(_snapshot(live_files), walk(live)[0])
|
|
if why:
|
|
raise RuntimeError(f"{why} while BPRT.new was built; nothing was replaced, "
|
|
"validate the batch again")
|
|
progress.step("Swapping BPRT.new in")
|
|
archive = _swap(new)
|
|
except BaseException:
|
|
if os.path.isdir(live): # never drop BPRT.new while it is the only full copy
|
|
shutil.rmtree(new, ignore_errors=True)
|
|
raise
|
|
progress.step("Recording")
|
|
entry = {"id": _stamp(), "at": uc.now(), "kind": "activate", "batch": name,
|
|
"archive": os.path.basename(archive), "sync": res.get("sync", {}),
|
|
"counts": res.get("counts", {}), "ppkg": res.get("ppkg", []),
|
|
"bom_fix": res.get("bom_fix", [])}
|
|
_add_history(entry)
|
|
shutil.rmtree(folder, ignore_errors=True)
|
|
try:
|
|
os.remove(os.path.join(_state("batches"), f"{name}.json"))
|
|
except OSError:
|
|
pass
|
|
try:
|
|
manifest(build=True) # seeds the cache for the BOM-fixed files
|
|
except Exception:
|
|
pass
|
|
_audit("BPRT_ACTIVATE", f"{name}: {res['counts']}; previous BPRT in {entry['archive']}")
|
|
return entry
|
|
|
|
|
|
# --- archives and rollback ---------------------------------------------------
|
|
|
|
def archives():
|
|
"""Full BPRT copies in BPRT-archive, newest first, with the bytes only
|
|
they hold (files not hard-linked into the live BPRT)."""
|
|
root = archive_root()
|
|
out = []
|
|
if not os.path.isdir(root):
|
|
return out
|
|
live_inodes = {(st.st_dev, st.st_ino) for st in walk(bprt_dir())[0].values()}
|
|
for name in os.listdir(root):
|
|
folder = os.path.join(root, name)
|
|
if not _ARCHIVE.match(name) or not os.path.isdir(folder) or os.path.islink(folder):
|
|
continue
|
|
files, _ = walk(folder)
|
|
view = vendor_view(folder, files)
|
|
out.append({
|
|
"name": name, "count": len(files),
|
|
"bytes": sum(st.st_size for st in files.values()),
|
|
"own_bytes": sum(st.st_size for st in files.values()
|
|
if (st.st_dev, st.st_ino) not in live_inodes),
|
|
"modified": os.stat(folder).st_mtime,
|
|
"packages": sorted(n for n in files if mcl_ppkg.is_mcl_name(n)),
|
|
"missing_required": [r for r in REQUIRED if r not in view],
|
|
})
|
|
out.sort(key=lambda a: a["name"], reverse=True)
|
|
return out
|
|
|
|
|
|
def _check_archive(name):
|
|
if not _ARCHIVE.match(str(name or "")):
|
|
raise ValueError(f"{name or '(blank)'} is not a BPRT archive")
|
|
folder = os.path.join(archive_root(), name)
|
|
if not os.path.isdir(folder) or os.path.islink(folder):
|
|
raise ValueError(f"{name} is not in BPRT-archive")
|
|
return folder
|
|
|
|
|
|
def rollback(name):
|
|
"""Make an archived BPRT live again; the current one is archived first,
|
|
so a rollback can itself be rolled back."""
|
|
folder = _check_archive(name)
|
|
if busy():
|
|
raise RuntimeError(f"a BPRT job ({busy()}) is running")
|
|
with _Lock():
|
|
_recover()
|
|
files, _ = walk(folder)
|
|
errors = []
|
|
check_packages(list(files), errors, [])
|
|
if errors:
|
|
raise ValueError("; ".join(errors))
|
|
archive = _swap(folder)
|
|
entry = {"id": _stamp(), "at": uc.now(), "kind": "rollback", "restored": name,
|
|
"archive": os.path.basename(archive)}
|
|
_add_history(entry)
|
|
return entry
|
|
|
|
|
|
def delete_archive(name):
|
|
folder = _check_archive(name)
|
|
if busy():
|
|
raise RuntimeError(f"a BPRT job ({busy()}) is running")
|
|
with _Lock():
|
|
shutil.rmtree(folder)
|
|
return name
|
|
|
|
|
|
def delete_batch(name):
|
|
folder = os.path.join(incoming_root(), str(name or ""))
|
|
base = name[:-5] if str(name).endswith(".part") else name
|
|
check_batch(base)
|
|
if not os.path.isdir(folder) or os.path.islink(folder):
|
|
raise ValueError(f"{name} is not an incoming batch")
|
|
st = job_state()
|
|
if st.get("running") and st.get("arg") == base:
|
|
raise RuntimeError(f"the {st.get('kind')} job is using this batch")
|
|
shutil.rmtree(folder)
|
|
if base == name: # X.part is an upload; the validation of a finished X stays
|
|
try:
|
|
os.remove(os.path.join(_state("batches"), f"{base}.json"))
|
|
except OSError:
|
|
pass
|
|
return name
|
|
|
|
|
|
def history():
|
|
return uc.read_json(_path("history.json"), []) or []
|
|
|
|
|
|
def _add_history(entry):
|
|
hist = history()
|
|
hist.insert(0, entry)
|
|
uc.write_json(_path("history.json"), hist[:100])
|
|
|
|
|
|
# --- zip upload fallback -----------------------------------------------------
|
|
|
|
def clean_zip_name(name):
|
|
name = os.path.basename(str(name or "").replace("\\", "/")).strip()
|
|
if not name.lower().endswith(".zip") or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9 ._()+-]{0,150}", name):
|
|
raise ValueError("upload a .zip file (letters, digits, spaces, . _ - ( ) + in the name)")
|
|
return name
|
|
|
|
|
|
def save_zip(stream, name, length=None):
|
|
"""Stream an uploaded zip to BPRT-incoming/.uploads (same filesystem as
|
|
BPRT, never in memory). Returns (name, bytes)."""
|
|
name = clean_zip_name(name)
|
|
ensure_incoming()
|
|
folder = uploads_dir()
|
|
os.makedirs(folder, exist_ok=True)
|
|
free = shutil.disk_usage(folder).free
|
|
if length and length * 2 > free - (1 << 30):
|
|
raise RuntimeError(f"not enough disk space: the zip and its unpacked files need about "
|
|
f"{2 * length // (1 << 20)} MB, {free // (1 << 20)} MB free")
|
|
part = os.path.join(folder, f".{name}.{os.getpid()}.part")
|
|
written = 0
|
|
try:
|
|
with open(part, "wb") as f:
|
|
while True:
|
|
block = stream.read(1 << 20)
|
|
if not block:
|
|
break
|
|
f.write(block)
|
|
written += len(block)
|
|
if length and written != length:
|
|
raise RuntimeError(f"upload incomplete: {written} of {length} bytes")
|
|
if not written:
|
|
raise RuntimeError("empty upload")
|
|
if not zipfile.is_zipfile(part):
|
|
raise ValueError(f"{name} is not a zip file")
|
|
os.replace(part, os.path.join(folder, name))
|
|
finally:
|
|
if os.path.exists(part):
|
|
os.remove(part)
|
|
return name, written
|
|
|
|
|
|
def uploaded_zips():
|
|
folder = uploads_dir()
|
|
if not os.path.isdir(folder):
|
|
return []
|
|
out = []
|
|
for n in sorted(os.listdir(folder)):
|
|
p = os.path.join(folder, n)
|
|
if n.lower().endswith(".zip") and not n.startswith(".") and os.path.isfile(p):
|
|
st = os.stat(p)
|
|
out.append({"name": n, "size": st.st_size, "modified": st.st_mtime})
|
|
return out
|
|
|
|
|
|
def delete_zip(name):
|
|
path = os.path.join(uploads_dir(), clean_zip_name(name))
|
|
if not os.path.isfile(path):
|
|
raise ValueError(f"{name} is not an uploaded zip")
|
|
os.remove(path)
|
|
|
|
|
|
def _zip_members(zf):
|
|
"""[(ZipInfo, relpath)] of the files in a zip, any single top folder
|
|
(e.g. BPRT/) stripped; ValueError on unsafe names."""
|
|
rows = []
|
|
for info in zf.infolist():
|
|
if info.is_dir():
|
|
continue
|
|
name = info.filename.replace("\\", "/")
|
|
if name.startswith("__MACOSX/") or name.rsplit("/", 1)[-1] in ("Thumbs.db", ".DS_Store"):
|
|
continue
|
|
if name.startswith("/") or re.match(r"^[A-Za-z]:", name):
|
|
raise ValueError(f"zip entry {info.filename!r} has an absolute path")
|
|
rows.append([info, name])
|
|
tops = {r[1].split("/", 1)[0] for r in rows}
|
|
if len(tops) == 1 and all("/" in r[1] for r in rows):
|
|
cut = len(next(iter(tops))) + 1
|
|
for r in rows:
|
|
r[1] = r[1][cut:]
|
|
for r in rows:
|
|
_check_rel(r[1])
|
|
return [(info, rel) for info, rel in rows]
|
|
|
|
|
|
def unzip(name, progress=None):
|
|
"""Unpack an uploaded zip into a new incoming batch. A zip made by
|
|
Sync-BPRT.ps1 -ZipOut carries its sync.json; any other zip is taken as the
|
|
complete vendor folder and gets a sync.json listing everything in it."""
|
|
progress = progress or Progress()
|
|
name = clean_zip_name(name)
|
|
src = os.path.join(uploads_dir(), name)
|
|
if not os.path.isfile(src):
|
|
raise ValueError(f"{name} is not an uploaded zip")
|
|
base = f"{_stamp()}-zip"
|
|
final = _unique(incoming_root(), base)
|
|
part = final + ".part"
|
|
os.makedirs(part)
|
|
try:
|
|
with zipfile.ZipFile(src) as zf:
|
|
members = _zip_members(zf)
|
|
if not members:
|
|
raise ValueError(f"{name} holds no files")
|
|
need = sum(i.file_size for i, _ in members)
|
|
free = shutil.disk_usage(incoming_root()).free
|
|
if need > free - (1 << 30): # keep 1 GB for the share
|
|
raise RuntimeError(f"not enough disk space: {name} unpacks to {need // (1 << 20)} MB, "
|
|
f"{free // (1 << 20)} MB free")
|
|
progress.start(sum(i.file_size for i, _ in members), len(members))
|
|
listed = []
|
|
for info, rel in members:
|
|
progress.current(rel)
|
|
dest = os.path.join(part, *rel.split("/"))
|
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
h = hashlib.sha256()
|
|
with zf.open(info) as fin, open(dest, "wb") as fout:
|
|
for block in iter(lambda: fin.read(1 << 22), b""):
|
|
fout.write(block)
|
|
h.update(block)
|
|
progress.add(len(block))
|
|
if rel != "sync.json":
|
|
listed.append({"path": rel, "size": info.file_size, "sha256": h.hexdigest().upper()})
|
|
if not os.path.exists(os.path.join(part, "sync.json")):
|
|
uc.write_json(os.path.join(part, "sync.json"), {
|
|
"format": 1, "tool": "webapp zip upload", "source": name,
|
|
"computer": "", "user": "", "created": uc.now(),
|
|
"files": listed, "uploaded": [r["path"] for r in listed],
|
|
})
|
|
os.rename(part, final)
|
|
except BaseException:
|
|
shutil.rmtree(part, ignore_errors=True)
|
|
raise
|
|
os.remove(src) # unpacked; the zip would only double the space used
|
|
_audit("BPRT_UNZIP", f"{name} into {os.path.basename(final)}")
|
|
return os.path.basename(final)
|
|
|
|
|
|
# --- imaging in progress -----------------------------------------------------
|
|
|
|
def _epoch(iso):
|
|
try:
|
|
dt = datetime.fromisoformat(str(iso))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if not dt.tzinfo:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return dt.timestamp()
|
|
|
|
|
|
def active_imaging():
|
|
"""Imaging sessions still running (pushed or inferred from the logs in the
|
|
last few hours): each of those PCs copies BPRT while it images."""
|
|
try:
|
|
from services import imaging_status
|
|
sessions = imaging_status.list_sessions()
|
|
window = imaging_status.ACTIVE_WINDOW_HOURS * 3600
|
|
except Exception:
|
|
return []
|
|
cutoff = time.time() - window
|
|
out = []
|
|
for s in sessions:
|
|
if s.get("status") in ("succeeded", "failed"):
|
|
continue
|
|
ts = _epoch(s.get("last_updated"))
|
|
if ts is None or ts < cutoff:
|
|
continue
|
|
out.append({"id": s.get("serial") or s.get("mac") or s.get("ip") or "?",
|
|
"hostname": s.get("hostname") or "", "stage": s.get("current_stage") or "",
|
|
"last_updated": s.get("last_updated")})
|
|
return out
|
|
|
|
|
|
def sync_script():
|
|
"""Path of static/Sync-BPRT.ps1 (shipped with the webapp) when it can be served."""
|
|
here = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
for p in (os.environ.get("BPRT_SYNC_SCRIPT", ""),
|
|
os.path.join(here, "static", "Sync-BPRT.ps1")):
|
|
if p and os.path.isfile(p):
|
|
return os.path.abspath(p)
|
|
return None
|
|
|
|
|
|
def _audit(action, detail):
|
|
try:
|
|
from services.audit import audit
|
|
audit(action, detail)
|
|
except Exception:
|
|
pass # audit log not writable (local tests): history.json still has it
|
|
|
|
|
|
# --- job entry point ---------------------------------------------------------
|
|
|
|
def main(argv):
|
|
if not argv or argv[0] not in ("manifest", "validate", "activate", "unzip"):
|
|
print("usage: python3 -m services.bprt_sync manifest | validate BATCH | activate BATCH | unzip NAME",
|
|
file=sys.stderr)
|
|
return 2
|
|
print(run_job(argv[0], argv[1] if len(argv) > 1 else ""))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|