webapp: imaging UX overhaul + image management CRUD
Imaging dashboard
- services/imaging_log_tail.py: parses dnsmasq leases, Apache access log,
Samba per-host log files, and dnsmasq syslog (DHCP/TFTP). Synthesizes
inferred sessions keyed by MAC for bays that have only touched the boot
chain but not yet pushed to /imaging/status. Active window 90 min.
- imaging_status.list_sessions() merges inferred sessions into the dashboard
list. Real client-pushed sessions win for the same MAC.
- imaging_status: stage_history field tracks every stage transition (capped
30); sidecar .log file per serial records every log_lines push uncapped
(read_full_log() caps detail-page response to 1 MB).
- delete_session/delete_all_sessions clean up sidecar .log too.
- New SSE endpoint /imaging/stream emits a session-list hash every 5s.
Client fetches /imaging/tiles (HTML partial) on hash change and swaps
#imaging-tiles innerHTML. Polling fallback at 15s if SSE drops.
- Tile-swap preserves scroll, filter input, expanded state via localStorage,
and any LAPS input the operator is mid-pasting (swap skipped when a
laps-input is focused).
- imaging.html: removed 15s location.reload(). Added live-status dot in
header (gray idle / green SSE connected / red SSE lost).
- _imaging_tiles.html: shared partial used by both /imaging full render and
/imaging/tiles SSE refresh. Inferred bays render with yellow border +
log-inferred badge + no progress bar (stage_index inference is coarse).
- imaging_detail.html (new): per-bay forensics page at /imaging/session/
<serial>. Session metadata grid, stage timeline table, full sidecar log
with truncation indicator, Copy-support-summary button. Linked from each
client-pushed tile.
- qr-render.js exposes window.renderAllQRs() so the SSE swap can re-render
Intune device-ID QRs in the swapped-in tiles.
Image management
- services/image_registry.py: JSON registry of image types at
{SAMBA_SHARE}/image-registry.json. Bootstraps from baked-in
config.IMAGE_TYPES on first run. create/clone/delete/rename_friendly
mutate the file then call reload() which rewrites config.IMAGE_TYPES +
config.FRIENDLY_NAMES in place. Sidebar reflects on next request.
- app.py routes: /images/new, /images/<t>/clone, /images/<t>/delete (with
optional content-wipe checkbox), /images/<t>/rename.
- dashboard.html: + New image type button + Clone/Delete per row, all in
Bootstrap modals with confirmation copy.
- Clone copies Deploy/ tree but preserves symlinks to shared dirs (Out-of-
box Drivers, Operating Systems, Packages) so disk usage stays low.
- Delete with content checked unlinks symlinks (does not follow into shared
dirs).
Driver / package upload + orphan adoption
- services/images.py: upload_driver, adopt_orphan, remove_orphans,
upload_package. Filename sanitization blocks path traversal.
- app.py routes: /images/<t>/drivers/upload, /images/<t>/drivers/adopt,
/images/<t>/drivers/orphans/delete, /images/<t>/packages/upload.
- image_config.html: Upload .zip button + modal on Drivers section. Orphan
drivers card-footer rebuilt as interactive list with per-row Adopt inline
form (family + destinationDir inputs) and bulk select+delete.
- Upload .zip on Packages section with optional destinationDir field that
appends a packages.json entry.
Configuration
- config.py: new env vars DNSMASQ_LEASES, APACHE_ACCESS_LOG, SAMBA_LOG_DIR,
DNSMASQ_SYSLOG for the log-tailer.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
cproudlock
254
webapp/services/image_registry.py
Normal file
254
webapp/services/image_registry.py
Normal file
@@ -0,0 +1,254 @@
|
||||
"""Mutable registry for image types.
|
||||
|
||||
Previously IMAGE_TYPES + FRIENDLY_NAMES were hardcoded in config.py and
|
||||
adding a new image type required a code edit + Ansible re-run. Now they
|
||||
live in a JSON file (REGISTRY_PATH) that the webapp can read and write at
|
||||
runtime.
|
||||
|
||||
On import (called from app.py startup), reload() rewrites
|
||||
config.IMAGE_TYPES and config.FRIENDLY_NAMES in-place so the rest of the
|
||||
code (and the base.html sidebar context) sees the live list without
|
||||
threading the registry through every call.
|
||||
|
||||
Schema:
|
||||
{
|
||||
"image_types": [
|
||||
{"key": "gea-standard", "friendly_name": "GE Aerospace Standard"},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
The seed comes from whatever IMAGE_TYPES/FRIENDLY_NAMES are baked into
|
||||
config.py at first run. Once the registry file exists, it wins.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import config
|
||||
|
||||
# Persist alongside the Samba share so an Ansible re-run that recreates
|
||||
# SAMBA_SHARE keeps the registry. Override via env for tests.
|
||||
REGISTRY_PATH = os.environ.get(
|
||||
"IMAGE_REGISTRY_PATH",
|
||||
os.path.join(config.SAMBA_SHARE, "image-registry.json"),
|
||||
)
|
||||
|
||||
# image_type keys are used as directory names + URL path components. Match
|
||||
# the existing baked-in convention (lowercase + hyphens + alnum).
|
||||
_VALID_KEY = re.compile(r"^[a-z][a-z0-9-]{1,63}$")
|
||||
|
||||
|
||||
class RegistryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _read_file() -> dict:
|
||||
try:
|
||||
with open(REGISTRY_PATH, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _write_file(data: dict):
|
||||
os.makedirs(os.path.dirname(REGISTRY_PATH), exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(
|
||||
dir=os.path.dirname(REGISTRY_PATH),
|
||||
prefix=".tmp-registry-",
|
||||
suffix=".json",
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
os.replace(tmp, REGISTRY_PATH)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _seed_from_config() -> dict:
|
||||
"""First-run bootstrap: build a registry dict from whatever's baked into
|
||||
config.py. Preserves order so the sidebar layout doesn't shuffle on the
|
||||
first reload."""
|
||||
items = []
|
||||
for key in config.IMAGE_TYPES:
|
||||
items.append({
|
||||
"key": key,
|
||||
"friendly_name": config.FRIENDLY_NAMES.get(key, key),
|
||||
"created_at": None,
|
||||
})
|
||||
return {"image_types": items}
|
||||
|
||||
|
||||
def load_registry() -> dict:
|
||||
"""Return the registry dict. Bootstraps from config.py on first read."""
|
||||
data = _read_file()
|
||||
if not data.get("image_types"):
|
||||
data = _seed_from_config()
|
||||
try:
|
||||
_write_file(data)
|
||||
except OSError:
|
||||
# Read-only filesystem / no perms; fall back to in-memory seed.
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def reload():
|
||||
"""Refresh config.IMAGE_TYPES + config.FRIENDLY_NAMES from the registry
|
||||
file. Called at startup and after every CRUD op. Mutates in place so
|
||||
existing references stay valid."""
|
||||
data = load_registry()
|
||||
keys = [item["key"] for item in data.get("image_types", []) if "key" in item]
|
||||
friendly = {item["key"]: item.get("friendly_name", item["key"])
|
||||
for item in data.get("image_types", [])}
|
||||
# In-place replace so module-level references still point at the same
|
||||
# list/dict objects.
|
||||
config.IMAGE_TYPES[:] = keys
|
||||
config.FRIENDLY_NAMES.clear()
|
||||
config.FRIENDLY_NAMES.update(friendly)
|
||||
|
||||
|
||||
def _validate_key(key: str):
|
||||
if not key or not _VALID_KEY.match(key):
|
||||
raise RegistryError(
|
||||
f"Invalid image_type key '{key}'. Must be lowercase alphanumeric "
|
||||
f"+ hyphens, 2-64 chars, starting with a letter."
|
||||
)
|
||||
|
||||
|
||||
def create(key: str, friendly_name: str) -> dict:
|
||||
"""Add a new image type with no on-disk content. Caller is responsible
|
||||
for populating Deploy/ via the import flow afterwards."""
|
||||
_validate_key(key)
|
||||
if not friendly_name:
|
||||
friendly_name = key
|
||||
data = load_registry()
|
||||
existing_keys = {item["key"] for item in data["image_types"]}
|
||||
if key in existing_keys:
|
||||
raise RegistryError(f"image_type '{key}' already exists")
|
||||
data["image_types"].append({
|
||||
"key": key,
|
||||
"friendly_name": friendly_name,
|
||||
"created_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
})
|
||||
_write_file(data)
|
||||
reload()
|
||||
return {"key": key, "friendly_name": friendly_name}
|
||||
|
||||
|
||||
def clone(src_key: str, dst_key: str, friendly_name: Optional[str] = None) -> dict:
|
||||
"""Duplicate a source image type's on-disk content + register the new
|
||||
key. Copies Deploy/ tree (which contains Control/ JSON configs +
|
||||
unattend XML), Tools/ if present, and the top-level Sources/ symlink if
|
||||
present. Shared dirs (Out-of-box Drivers, Operating Systems, Packages)
|
||||
are re-symlinked into the new image root, not duplicated."""
|
||||
_validate_key(dst_key)
|
||||
data = load_registry()
|
||||
keys = {item["key"] for item in data["image_types"]}
|
||||
if src_key not in keys:
|
||||
raise RegistryError(f"source image_type '{src_key}' not registered")
|
||||
if dst_key in keys:
|
||||
raise RegistryError(f"destination image_type '{dst_key}' already exists")
|
||||
|
||||
from services import fs # local import to avoid circular at module load
|
||||
src_root = fs.image_root(src_key)
|
||||
dst_root = fs.image_root(dst_key)
|
||||
if os.path.exists(dst_root):
|
||||
raise RegistryError(f"destination path already exists on disk: {dst_root}")
|
||||
|
||||
os.makedirs(dst_root, exist_ok=True)
|
||||
if os.path.isdir(src_root):
|
||||
for entry in os.listdir(src_root):
|
||||
src_item = os.path.join(src_root, entry)
|
||||
dst_item = os.path.join(dst_root, entry)
|
||||
if os.path.islink(src_item):
|
||||
# Preserve symlink (shared dir) instead of dereferencing.
|
||||
target = os.readlink(src_item)
|
||||
os.symlink(target, dst_item)
|
||||
elif os.path.isdir(src_item):
|
||||
shutil.copytree(src_item, dst_item, symlinks=True)
|
||||
else:
|
||||
shutil.copy2(src_item, dst_item)
|
||||
|
||||
if not friendly_name:
|
||||
src_friendly = next(
|
||||
(i["friendly_name"] for i in data["image_types"] if i["key"] == src_key),
|
||||
src_key,
|
||||
)
|
||||
friendly_name = f"{src_friendly} (copy)"
|
||||
|
||||
data["image_types"].append({
|
||||
"key": dst_key,
|
||||
"friendly_name": friendly_name,
|
||||
"created_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"cloned_from": src_key,
|
||||
})
|
||||
_write_file(data)
|
||||
reload()
|
||||
return {"key": dst_key, "friendly_name": friendly_name, "cloned_from": src_key}
|
||||
|
||||
|
||||
def delete(key: str, delete_content: bool = False) -> dict:
|
||||
"""Remove key from the registry. When delete_content is True, also wipes
|
||||
the on-disk Deploy/Tools/etc tree. Shared symlinked dirs are NOT
|
||||
followed when deleting content (we unlink the symlink, not its target)."""
|
||||
data = load_registry()
|
||||
items = data.get("image_types", [])
|
||||
found = [i for i in items if i["key"] == key]
|
||||
if not found:
|
||||
raise RegistryError(f"image_type '{key}' not registered")
|
||||
data["image_types"] = [i for i in items if i["key"] != key]
|
||||
_write_file(data)
|
||||
reload()
|
||||
|
||||
removed_content = False
|
||||
if delete_content:
|
||||
from services import fs
|
||||
root = fs.image_root(key)
|
||||
if os.path.isdir(root):
|
||||
# Walk top-level: unlink symlinks (don't follow into shared
|
||||
# dirs), rmtree real directories, unlink files.
|
||||
for entry in os.listdir(root):
|
||||
p = os.path.join(root, entry)
|
||||
try:
|
||||
if os.path.islink(p):
|
||||
os.unlink(p)
|
||||
elif os.path.isdir(p):
|
||||
shutil.rmtree(p)
|
||||
else:
|
||||
os.unlink(p)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.rmdir(root)
|
||||
except OSError:
|
||||
pass
|
||||
removed_content = True
|
||||
|
||||
return {"key": key, "removed_content": removed_content}
|
||||
|
||||
|
||||
def rename_friendly(key: str, friendly_name: str) -> dict:
|
||||
"""Update the human-readable name without touching the key or on-disk
|
||||
content. Sidebar reflects on next request."""
|
||||
if not friendly_name:
|
||||
raise RegistryError("friendly_name required")
|
||||
data = load_registry()
|
||||
for item in data.get("image_types", []):
|
||||
if item["key"] == key:
|
||||
item["friendly_name"] = friendly_name
|
||||
_write_file(data)
|
||||
reload()
|
||||
return {"key": key, "friendly_name": friendly_name}
|
||||
raise RegistryError(f"image_type '{key}' not registered")
|
||||
@@ -1,11 +1,24 @@
|
||||
"""Per-image-type state probes: status + config (drivers / OS / packages / models)."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
|
||||
import config
|
||||
from services import fs
|
||||
|
||||
|
||||
# File-name sanitizer for uploads. Strips anything outside [A-Za-z0-9._-]
|
||||
# so a malicious filename can't break out of the destination directory.
|
||||
_SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]")
|
||||
|
||||
|
||||
def _safe_filename(name: str) -> str:
|
||||
base = os.path.basename(name or "")
|
||||
cleaned = _SAFE_NAME.sub("_", base)
|
||||
return cleaned or "upload.bin"
|
||||
|
||||
|
||||
def image_status(image_type):
|
||||
"""Return a dict describing the state of an image type."""
|
||||
dp = fs.deploy_path(image_type)
|
||||
@@ -121,3 +134,131 @@ def load_image_config(image_type):
|
||||
"orphan_drivers": orphan_drivers,
|
||||
"os_selection": os_selection,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver / package upload + orphan adoption
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DRIVERS_SUBDIR = "Out-of-box Drivers"
|
||||
PACKAGES_SUBDIR = "Packages"
|
||||
|
||||
|
||||
def _drivers_dir(image_type: str) -> str:
|
||||
return os.path.join(fs.deploy_path(image_type), DRIVERS_SUBDIR)
|
||||
|
||||
|
||||
def _packages_dir(image_type: str) -> str:
|
||||
return os.path.join(fs.deploy_path(image_type), PACKAGES_SUBDIR)
|
||||
|
||||
|
||||
def upload_driver(image_type: str, uploaded_file, family: str = "",
|
||||
destination_dir: str = "", overwrite: bool = False) -> dict:
|
||||
"""Save an uploaded driver .zip into the image's Out-of-box Drivers dir.
|
||||
When family + destination_dir are provided, also append a HardwareDriver
|
||||
.json entry so the driver is recognized at deploy time. Returns a dict
|
||||
describing what landed on disk + whether an entry was registered."""
|
||||
fname = _safe_filename(uploaded_file.filename)
|
||||
if not fname.lower().endswith(".zip"):
|
||||
raise ValueError("Driver upload must be a .zip file")
|
||||
dst_dir = _drivers_dir(image_type)
|
||||
os.makedirs(dst_dir, exist_ok=True)
|
||||
dst_path = os.path.join(dst_dir, fname)
|
||||
if os.path.exists(dst_path) and not overwrite:
|
||||
raise FileExistsError(f"{fname} already exists in {DRIVERS_SUBDIR}/")
|
||||
uploaded_file.save(dst_path)
|
||||
registered = False
|
||||
if family and destination_dir:
|
||||
adopt_orphan(image_type, fname, family, destination_dir)
|
||||
registered = True
|
||||
return {
|
||||
"filename": fname,
|
||||
"path": dst_path,
|
||||
"registered": registered,
|
||||
}
|
||||
|
||||
|
||||
def adopt_orphan(image_type: str, filename: str, family: str,
|
||||
destination_dir: str) -> dict:
|
||||
"""Append a HardwareDriver.json entry for an existing .zip in the image's
|
||||
Out-of-box Drivers dir, so it stops showing up as an orphan + gets
|
||||
deployed for the named hardware family. Idempotent: a second adopt with
|
||||
the same filename is a no-op."""
|
||||
safe_name = _safe_filename(filename)
|
||||
drivers_path = _drivers_dir(image_type)
|
||||
if not os.path.isfile(os.path.join(drivers_path, safe_name)):
|
||||
raise FileNotFoundError(f"{safe_name} not found in {DRIVERS_SUBDIR}/")
|
||||
if not family:
|
||||
raise ValueError("family is required (matches HardwareModelSelection.Id)")
|
||||
if not destination_dir:
|
||||
raise ValueError("destination_dir is required")
|
||||
|
||||
ctrl = fs.control_path(image_type)
|
||||
hw_file = os.path.join(ctrl, "HardwareDriver.json")
|
||||
entries = fs.load_json(hw_file)
|
||||
for e in entries:
|
||||
if (e.get("FileName") or e.get("fileName") or "").lower() == safe_name.lower():
|
||||
return {"filename": safe_name, "already_registered": True}
|
||||
entries.append({
|
||||
"FileName": safe_name,
|
||||
"DestinationDir": destination_dir,
|
||||
"family": family,
|
||||
})
|
||||
fs.save_json(hw_file, entries)
|
||||
return {"filename": safe_name, "already_registered": False}
|
||||
|
||||
|
||||
def remove_orphans(image_type: str, filenames: list[str]) -> dict:
|
||||
"""Delete the named files from Out-of-box Drivers/. Caller is
|
||||
responsible for confirming this is what the user wants (orphan files
|
||||
have no JSON entry, so deleting them is safe in the sense that nothing
|
||||
references them; but they may be stash for future adoption)."""
|
||||
drivers_path = _drivers_dir(image_type)
|
||||
removed, missing = [], []
|
||||
for fn in filenames:
|
||||
safe = _safe_filename(fn)
|
||||
path = os.path.join(drivers_path, safe)
|
||||
if not os.path.isfile(path):
|
||||
missing.append(safe)
|
||||
continue
|
||||
try:
|
||||
os.unlink(path)
|
||||
removed.append(safe)
|
||||
except OSError:
|
||||
missing.append(safe)
|
||||
return {"removed": removed, "missing": missing}
|
||||
|
||||
|
||||
def upload_package(image_type: str, uploaded_file, destination_dir: str = "",
|
||||
overwrite: bool = False) -> dict:
|
||||
"""Save an uploaded package (any extension) into the image's Packages
|
||||
dir, and append an entry to packages.json when destination_dir is set
|
||||
so it deploys at imaging time."""
|
||||
fname = _safe_filename(uploaded_file.filename)
|
||||
dst_dir = _packages_dir(image_type)
|
||||
os.makedirs(dst_dir, exist_ok=True)
|
||||
dst_path = os.path.join(dst_dir, fname)
|
||||
if os.path.exists(dst_path) and not overwrite:
|
||||
raise FileExistsError(f"{fname} already exists in {PACKAGES_SUBDIR}/")
|
||||
uploaded_file.save(dst_path)
|
||||
registered = False
|
||||
if destination_dir:
|
||||
ctrl = fs.control_path(image_type)
|
||||
pkg_file = os.path.join(ctrl, "packages.json")
|
||||
entries = fs.load_json(pkg_file)
|
||||
already = any(
|
||||
(e.get("fileName") or e.get("FileName") or "").lower() == fname.lower()
|
||||
for e in entries
|
||||
)
|
||||
if not already:
|
||||
entries.append({
|
||||
"fileName": fname,
|
||||
"destinationDir": destination_dir,
|
||||
})
|
||||
fs.save_json(pkg_file, entries)
|
||||
registered = True
|
||||
return {
|
||||
"filename": fname,
|
||||
"path": dst_path,
|
||||
"registered": registered,
|
||||
}
|
||||
|
||||
376
webapp/services/imaging_log_tail.py
Normal file
376
webapp/services/imaging_log_tail.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""Infer in-progress imaging sessions from server-side logs.
|
||||
|
||||
The /imaging/status endpoint only sees clients that have reached the WinPE
|
||||
startnet.cmd push (stage_index 2). A bay that gets stuck earlier (no DHCP,
|
||||
TFTP fail, boot.wim download stall) is invisible to the operator.
|
||||
|
||||
This module tails dnsmasq leases + Apache access log + per-host Samba logs
|
||||
and synthesizes a "session" record per active MAC, with a coarse inferred
|
||||
stage based on which boot assets the client has actually fetched. Output
|
||||
shape matches services/imaging_status so list_sessions() can merge both.
|
||||
|
||||
Inferred sessions carry source="inferred" and never overwrite a real
|
||||
client-pushed session for the same serial (correlated by MAC when present).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import config
|
||||
|
||||
# Active window: an inferred session is shown if any evidence is newer than
|
||||
# this many seconds. Past that we assume the bay is idle / done / off.
|
||||
INFERRED_ACTIVE_WINDOW_S = 90 * 60 # 90 min
|
||||
|
||||
# Tail size caps so a giant log doesn't pull the whole file into memory on
|
||||
# each dashboard refresh.
|
||||
APACHE_TAIL_BYTES = 512 * 1024 # 512 KB
|
||||
SAMBA_TAIL_BYTES = 64 * 1024 # 64 KB per file
|
||||
SYSLOG_TAIL_BYTES = 256 * 1024 # 256 KB
|
||||
|
||||
# Path prefixes Apache serves, mapped to coarse imaging stage signals.
|
||||
# Order matters: we take the highest-numbered match (latest in the boot chain).
|
||||
_APACHE_STAGE_HITS = [
|
||||
("/menu.ipxe", ("stage_0_menu", 0)),
|
||||
("/win11/boot/", ("stage_0_boot_pre", 0)),
|
||||
("/win11/efi/", ("stage_0_boot_pre", 0)),
|
||||
("/win11/sources/boot.wim", ("stage_1_wim_get", 1)),
|
||||
("/win11/sources/", ("stage_1_wim_get", 1)),
|
||||
]
|
||||
|
||||
# TFTP bootloader fetches arrive via dnsmasq, not Apache. Mapped same way.
|
||||
_TFTP_STAGE_HITS = [
|
||||
("undionly.kpxe", ("stage_0_tftp_bios", 0)),
|
||||
("ipxe.efi", ("stage_0_tftp_uefi", 0)),
|
||||
]
|
||||
|
||||
# Apache combined log: <ip> - - [DD/Mon/YYYY:HH:MM:SS +ZZZZ] "GET /path HTTP/x.y" status bytes "ref" "ua"
|
||||
_APACHE_RE = re.compile(
|
||||
r'^(?P<ip>\S+)\s+\S+\s+\S+\s+'
|
||||
r'\[(?P<ts>[^\]]+)\]\s+'
|
||||
r'"(?P<method>\S+)\s+(?P<path>\S+)\s+\S+"\s+'
|
||||
r'(?P<status>\d+)\s+(?P<bytes>\S+)'
|
||||
)
|
||||
_APACHE_TS_FMT = "%d/%b/%Y:%H:%M:%S %z"
|
||||
|
||||
# dnsmasq syslog: "<MMM DD HH:MM:SS> host dnsmasq-tftp[pid]: sent /tftp/path to ip"
|
||||
# or "dnsmasq-dhcp[pid]: DHCPACK(...) ip mac hostname"
|
||||
_SYSLOG_DNSMASQ_RE = re.compile(
|
||||
r'^(?P<mon>\w{3})\s+(?P<day>\d+)\s+(?P<time>\d{2}:\d{2}:\d{2})\s+'
|
||||
r'\S+\s+dnsmasq-(?P<kind>\w+)\[\d+\]:\s+(?P<msg>.+)$'
|
||||
)
|
||||
|
||||
|
||||
def _tail_bytes(path: str, max_bytes: int) -> str:
|
||||
"""Read last max_bytes of a text log, return decoded string. Returns
|
||||
empty string if file is missing / unreadable. Skips partial first line."""
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
return ""
|
||||
start = max(0, size - max_bytes)
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(start)
|
||||
data = f.read()
|
||||
except OSError:
|
||||
return ""
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
if start > 0:
|
||||
# Drop partial first line.
|
||||
nl = text.find("\n")
|
||||
if nl >= 0:
|
||||
text = text[nl + 1:]
|
||||
return text
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _epoch_to_iso(epoch: float) -> str:
|
||||
return datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def parse_leases() -> list[dict]:
|
||||
"""dnsmasq.leases format: <expires_epoch> <mac> <ip> <hostname> <client_id>."""
|
||||
out = []
|
||||
try:
|
||||
with open(config.DNSMASQ_LEASES, "r") as f:
|
||||
for line in f:
|
||||
parts = line.split()
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
try:
|
||||
expires = int(parts[0])
|
||||
except ValueError:
|
||||
continue
|
||||
out.append({
|
||||
"expires": expires,
|
||||
"mac": parts[1].lower(),
|
||||
"ip": parts[2],
|
||||
"hostname": parts[3] if parts[3] != "*" else "",
|
||||
})
|
||||
except OSError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def parse_apache_hits() -> dict:
|
||||
"""Return {ip: {"last_ts": epoch, "max_stage": int, "stage_label": str,
|
||||
"paths": [recent_paths]}}."""
|
||||
out: dict = {}
|
||||
text = _tail_bytes(config.APACHE_ACCESS_LOG, APACHE_TAIL_BYTES)
|
||||
for line in text.splitlines():
|
||||
m = _APACHE_RE.match(line)
|
||||
if not m:
|
||||
continue
|
||||
ip = m.group("ip")
|
||||
path = m.group("path").lower()
|
||||
try:
|
||||
ts = datetime.strptime(m.group("ts"), _APACHE_TS_FMT).timestamp()
|
||||
except ValueError:
|
||||
continue
|
||||
# Match against known stage prefixes; keep the highest-numbered.
|
||||
for prefix, (label, stage) in _APACHE_STAGE_HITS:
|
||||
if path.startswith(prefix):
|
||||
rec = out.setdefault(ip, {
|
||||
"last_ts": 0.0, "max_stage": -1,
|
||||
"stage_label": "", "paths": [],
|
||||
})
|
||||
if ts > rec["last_ts"]:
|
||||
rec["last_ts"] = ts
|
||||
if stage > rec["max_stage"]:
|
||||
rec["max_stage"] = stage
|
||||
rec["stage_label"] = label
|
||||
rec["paths"].append(path)
|
||||
# Cap path list so a busy client doesn't explode the dict.
|
||||
rec["paths"] = rec["paths"][-10:]
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def parse_syslog_tftp() -> dict:
|
||||
"""Return {mac_or_ip: {"last_ts": epoch, "max_stage": int,
|
||||
"stage_label": str, "evidence": [str]}} from dnsmasq syslog lines."""
|
||||
out: dict = {}
|
||||
text = _tail_bytes(config.DNSMASQ_SYSLOG, SYSLOG_TAIL_BYTES)
|
||||
year = datetime.now().year
|
||||
for line in text.splitlines():
|
||||
m = _SYSLOG_DNSMASQ_RE.match(line)
|
||||
if not m:
|
||||
continue
|
||||
kind = m.group("kind")
|
||||
msg = m.group("msg")
|
||||
# Best-effort ts parse. Syslog format has no year; assume current.
|
||||
try:
|
||||
ts = datetime.strptime(
|
||||
f"{year} {m.group('mon')} {m.group('day')} {m.group('time')}",
|
||||
"%Y %b %d %H:%M:%S"
|
||||
).timestamp()
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if kind == "tftp":
|
||||
# "sent /srv/tftp/ipxe.efi to 172.16.9.42"
|
||||
tm = re.search(r'(?:sent|read|file)\s+(?P<file>\S+)\s+to\s+(?P<ip>\S+)', msg)
|
||||
if not tm:
|
||||
continue
|
||||
key = tm.group("ip")
|
||||
fname = os.path.basename(tm.group("file")).lower()
|
||||
for needle, (label, stage) in _TFTP_STAGE_HITS:
|
||||
if needle in fname:
|
||||
rec = out.setdefault(key, {
|
||||
"last_ts": 0.0, "max_stage": -1,
|
||||
"stage_label": "", "evidence": [],
|
||||
})
|
||||
if ts > rec["last_ts"]:
|
||||
rec["last_ts"] = ts
|
||||
if stage > rec["max_stage"]:
|
||||
rec["max_stage"] = stage
|
||||
rec["stage_label"] = label
|
||||
rec["evidence"].append(f"tftp:{fname}")
|
||||
rec["evidence"] = rec["evidence"][-5:]
|
||||
break
|
||||
elif kind == "dhcp":
|
||||
# DHCPACK lines confirm a lease handshake completed; useful as a
|
||||
# very-early "this MAC is alive" signal even before TFTP fires.
|
||||
dm = re.search(
|
||||
r'DHCP(?:ACK|OFFER|REQUEST)\([^)]+\)\s+'
|
||||
r'(?P<ip>\d+\.\d+\.\d+\.\d+)\s+'
|
||||
r'(?P<mac>[0-9a-f:]{17})',
|
||||
msg
|
||||
)
|
||||
if not dm:
|
||||
continue
|
||||
key = dm.group("ip")
|
||||
rec = out.setdefault(key, {
|
||||
"last_ts": 0.0, "max_stage": -1,
|
||||
"stage_label": "", "evidence": [],
|
||||
})
|
||||
if ts > rec["last_ts"]:
|
||||
rec["last_ts"] = ts
|
||||
if rec["max_stage"] < 0:
|
||||
rec["max_stage"] = 0
|
||||
rec["stage_label"] = "stage_0_dhcp"
|
||||
rec["evidence"].append(f"dhcp:{dm.group('mac')}")
|
||||
rec["evidence"] = rec["evidence"][-5:]
|
||||
return out
|
||||
|
||||
|
||||
def parse_samba_hits() -> dict:
|
||||
"""Return {ip: {"last_ts": epoch, "shares": set[str]}} for clients that
|
||||
have opened a Samba session in the recent tail. Samba writes per-host log
|
||||
files: /var/log/samba/log.<ip>. Mere existence + recent mtime = client
|
||||
has connected. Share names from inside the file are best-effort."""
|
||||
out: dict = {}
|
||||
log_dir = config.SAMBA_LOG_DIR
|
||||
if not os.path.isdir(log_dir):
|
||||
return out
|
||||
try:
|
||||
entries = os.listdir(log_dir)
|
||||
except OSError:
|
||||
return out
|
||||
for name in entries:
|
||||
if not name.startswith("log."):
|
||||
continue
|
||||
ip = name[len("log."):]
|
||||
# Skip the daemon's own log files (log.smbd, log.nmbd, log.winbindd).
|
||||
if ip in ("smbd", "nmbd", "winbindd") or "." not in ip:
|
||||
continue
|
||||
path = os.path.join(log_dir, name)
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
continue
|
||||
rec = {"last_ts": st.st_mtime, "shares": set()}
|
||||
text = _tail_bytes(path, SAMBA_TAIL_BYTES)
|
||||
for share in ("winpeapps", "clonezilla", "enrollment", "image-upload", "blancco-reports"):
|
||||
if f"[{share}]" in text or f"connect to service {share}" in text:
|
||||
rec["shares"].add(share)
|
||||
out[ip] = rec
|
||||
return out
|
||||
|
||||
|
||||
def infer_sessions(
|
||||
active_window_s: int = INFERRED_ACTIVE_WINDOW_S,
|
||||
) -> list[dict]:
|
||||
"""Build inferred-session records from all log sources. Keyed by MAC when
|
||||
available, else IP. Only returns sessions with evidence in the active
|
||||
window."""
|
||||
leases = parse_leases()
|
||||
apache = parse_apache_hits()
|
||||
samba = parse_samba_hits()
|
||||
tftp = parse_syslog_tftp()
|
||||
|
||||
# IP -> lease (MAC, hostname).
|
||||
by_ip = {lease["ip"]: lease for lease in leases}
|
||||
|
||||
cutoff = time.time() - active_window_s
|
||||
sessions: dict[str, dict] = {}
|
||||
|
||||
def _ensure(ip: str) -> dict:
|
||||
lease = by_ip.get(ip, {})
|
||||
key = lease.get("mac") or ip
|
||||
rec = sessions.get(key)
|
||||
if rec is None:
|
||||
rec = {
|
||||
"source": "inferred",
|
||||
"serial": None,
|
||||
"mac": lease.get("mac", ""),
|
||||
"ip": ip,
|
||||
"hostname_target": lease.get("hostname", ""),
|
||||
"started_at": None,
|
||||
"last_updated": None,
|
||||
"stage_index": 0,
|
||||
"stage_total": 8,
|
||||
"current_stage": "",
|
||||
"status": "in_progress",
|
||||
"evidence": [],
|
||||
"log_tail": [],
|
||||
}
|
||||
sessions[key] = rec
|
||||
return rec
|
||||
|
||||
def _bump_stage(rec: dict, stage: int, label: str):
|
||||
if stage > rec["stage_index"]:
|
||||
rec["stage_index"] = stage
|
||||
rec["current_stage"] = label
|
||||
|
||||
def _touch(rec: dict, ts: float):
|
||||
iso = _epoch_to_iso(ts)
|
||||
if rec["started_at"] is None or iso < rec["started_at"]:
|
||||
rec["started_at"] = iso
|
||||
if rec["last_updated"] is None or iso > rec["last_updated"]:
|
||||
rec["last_updated"] = iso
|
||||
|
||||
for ip, hit in tftp.items():
|
||||
if hit["last_ts"] < cutoff:
|
||||
continue
|
||||
rec = _ensure(ip)
|
||||
_touch(rec, hit["last_ts"])
|
||||
_bump_stage(rec, hit["max_stage"], hit["stage_label"])
|
||||
rec["evidence"].extend(hit["evidence"])
|
||||
|
||||
for ip, hit in apache.items():
|
||||
if hit["last_ts"] < cutoff:
|
||||
continue
|
||||
rec = _ensure(ip)
|
||||
_touch(rec, hit["last_ts"])
|
||||
_bump_stage(rec, hit["max_stage"], hit["stage_label"])
|
||||
for p in hit["paths"]:
|
||||
rec["evidence"].append(f"http:{p}")
|
||||
|
||||
for ip, hit in samba.items():
|
||||
if hit["last_ts"] < cutoff:
|
||||
continue
|
||||
rec = _ensure(ip)
|
||||
_touch(rec, hit["last_ts"])
|
||||
# WinPE-stage SMB activity = boot.wim applied + apps stage starting.
|
||||
_bump_stage(rec, 2, "stage_2_smb_winpe")
|
||||
for share in sorted(hit["shares"]):
|
||||
rec["evidence"].append(f"smb:{share}")
|
||||
|
||||
# Trim evidence + bake log_tail for display.
|
||||
for rec in sessions.values():
|
||||
rec["evidence"] = rec["evidence"][-15:]
|
||||
rec["log_tail"] = list(rec["evidence"])
|
||||
|
||||
return list(sessions.values())
|
||||
|
||||
|
||||
def merge_with_client_sessions(
|
||||
client_sessions: list[dict],
|
||||
inferred: Optional[list[dict]] = None,
|
||||
) -> list[dict]:
|
||||
"""Return a single list. Real (client-pushed) sessions win over inferred
|
||||
ones for the same MAC. Inferred sessions are dropped when a real session
|
||||
with the same MAC already exists, since the real one has strictly more
|
||||
information."""
|
||||
if inferred is None:
|
||||
inferred = infer_sessions()
|
||||
|
||||
real_macs = {(s.get("mac") or "").lower() for s in client_sessions if s.get("mac")}
|
||||
real_ips = {s.get("ip") for s in client_sessions if s.get("ip")}
|
||||
|
||||
# Tag real sessions so the template can style them differently from
|
||||
# inferred ones (and not break older sessions that lack the field).
|
||||
for s in client_sessions:
|
||||
s.setdefault("source", "client")
|
||||
|
||||
merged = list(client_sessions)
|
||||
for inf in inferred:
|
||||
mac = (inf.get("mac") or "").lower()
|
||||
ip = inf.get("ip")
|
||||
if mac and mac in real_macs:
|
||||
continue
|
||||
if ip and ip in real_ips:
|
||||
continue
|
||||
merged.append(inf)
|
||||
|
||||
merged.sort(key=lambda s: s.get("last_updated") or "", reverse=True)
|
||||
return merged
|
||||
@@ -20,8 +20,16 @@ from typing import Optional
|
||||
|
||||
import config
|
||||
|
||||
# How many recent log lines to keep per session.
|
||||
# How many recent log lines to keep per session in the JSON (dashboard tile
|
||||
# quick view). The full unbounded log is appended to a sidecar .log file
|
||||
# next to the .json so the detail page can show everything.
|
||||
LOG_TAIL_MAX = 50
|
||||
# Cap how many stage transitions we record per session (bounds JSON size on
|
||||
# pathological loops; 30 covers more than any real run uses).
|
||||
STAGE_HISTORY_MAX = 30
|
||||
# Detail page caps how many bytes of the sidecar .log it sends to the
|
||||
# browser, to avoid blowing up the response for a runaway log.
|
||||
DETAIL_LOG_MAX_BYTES = 1024 * 1024 # 1 MB
|
||||
# Sessions older than this are considered stale and dropped from the dashboard
|
||||
# "active" list. Still readable individually.
|
||||
ACTIVE_WINDOW_HOURS = 6
|
||||
@@ -40,6 +48,30 @@ def _path_for(serial: str) -> str:
|
||||
return os.path.join(config.IMAGING_DIR, f"{safe}.json")
|
||||
|
||||
|
||||
def _log_path_for(serial: str) -> str:
|
||||
safe = _SAFE_SERIAL.sub("_", serial.strip()) or "unknown"
|
||||
return os.path.join(config.IMAGING_DIR, f"{safe}.log")
|
||||
|
||||
|
||||
def _append_full_log(serial: str, lines):
|
||||
"""Best-effort append to the per-serial sidecar log file. Each line is
|
||||
timestamped. Failures are swallowed (status-tracking is not the
|
||||
authoritative log source; the .log is a convenience for the detail
|
||||
page)."""
|
||||
if not lines:
|
||||
return
|
||||
if isinstance(lines, str):
|
||||
lines = [lines]
|
||||
try:
|
||||
_ensure_dir()
|
||||
with open(_log_path_for(serial), "a") as f:
|
||||
ts = _now_iso()
|
||||
for line in lines:
|
||||
f.write(f"{ts} {line}\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
|
||||
@@ -100,7 +132,9 @@ def update_session(payload: dict) -> dict:
|
||||
# Fresh state after a rewind - mint a new started_at.
|
||||
state["started_at"] = _now_iso()
|
||||
|
||||
# Append any new log lines (preserve old; cap to LOG_TAIL_MAX).
|
||||
# Append any new log lines: capped tail in the JSON for the dashboard
|
||||
# quick view, and unbounded append to the sidecar .log for the detail
|
||||
# page.
|
||||
new_lines = payload.pop("log_lines", None)
|
||||
if new_lines:
|
||||
if isinstance(new_lines, str):
|
||||
@@ -108,6 +142,33 @@ def update_session(payload: dict) -> dict:
|
||||
tail = list(state.get("log_tail", []))
|
||||
tail.extend(new_lines)
|
||||
state["log_tail"] = tail[-LOG_TAIL_MAX:]
|
||||
_append_full_log(serial, new_lines)
|
||||
|
||||
# Stage history: record a transition row whenever stage_index increases
|
||||
# or status changes. Bounded to STAGE_HISTORY_MAX so a bouncing client
|
||||
# can't blow up the JSON. The dashboard tile only needs current state;
|
||||
# the detail page renders the timeline from this list.
|
||||
history = list(state.get("stage_history", []))
|
||||
try:
|
||||
new_idx = int(payload.get("stage_index") or 0)
|
||||
except (TypeError, ValueError):
|
||||
new_idx = 0
|
||||
try:
|
||||
old_idx = int(state.get("stage_index") or 0)
|
||||
except (TypeError, ValueError):
|
||||
old_idx = 0
|
||||
new_status = payload.get("status") or state.get("status") or "in_progress"
|
||||
old_status = state.get("status") or ""
|
||||
stage_changed = new_idx > old_idx
|
||||
status_changed = new_status != old_status and new_status in ("succeeded", "failed")
|
||||
if stage_changed or status_changed or not history:
|
||||
history.append({
|
||||
"ts": _now_iso(),
|
||||
"stage_index": new_idx or old_idx,
|
||||
"current_stage": payload.get("current_stage") or state.get("current_stage", ""),
|
||||
"status": new_status,
|
||||
})
|
||||
state["stage_history"] = history[-STAGE_HISTORY_MAX:]
|
||||
|
||||
for key, value in payload.items():
|
||||
if value is None or value == "":
|
||||
@@ -132,8 +193,14 @@ def update_session(payload: dict) -> dict:
|
||||
return state
|
||||
|
||||
|
||||
def list_sessions() -> list[dict]:
|
||||
"""Return all sessions sorted by last_updated desc."""
|
||||
def list_sessions(include_inferred: bool = True) -> list[dict]:
|
||||
"""Return all sessions sorted by last_updated desc.
|
||||
|
||||
When include_inferred is True (default for the dashboard), also pull
|
||||
server-side log-tail evidence and append synthesized sessions for any
|
||||
bay that has touched DHCP/TFTP/boot.wim but not yet pushed status.
|
||||
Real client-pushed sessions always win for the same MAC.
|
||||
"""
|
||||
_ensure_dir()
|
||||
out: list[dict] = []
|
||||
for name in os.listdir(config.IMAGING_DIR):
|
||||
@@ -145,7 +212,11 @@ def list_sessions() -> list[dict]:
|
||||
out.append(json.load(f))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
out.sort(key=lambda s: s.get("last_updated", ""), reverse=True)
|
||||
if include_inferred:
|
||||
from services import imaging_log_tail
|
||||
out = imaging_log_tail.merge_with_client_sessions(out)
|
||||
else:
|
||||
out.sort(key=lambda s: s.get("last_updated", ""), reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
@@ -160,27 +231,58 @@ def get_session(serial: str) -> Optional[dict]:
|
||||
return None
|
||||
|
||||
|
||||
def read_full_log(serial: str, max_bytes: int = DETAIL_LOG_MAX_BYTES) -> tuple[str, bool]:
|
||||
"""Return (text, truncated). Reads the trailing max_bytes of the sidecar
|
||||
.log. `truncated` is True when the file was larger than max_bytes and
|
||||
the leading slice was dropped."""
|
||||
path = _log_path_for(serial)
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
return ("", False)
|
||||
truncated = size > max_bytes
|
||||
start = max(0, size - max_bytes)
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(start)
|
||||
data = f.read()
|
||||
except OSError:
|
||||
return ("", False)
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
if truncated:
|
||||
nl = text.find("\n")
|
||||
if nl >= 0:
|
||||
text = text[nl + 1:]
|
||||
return (text, truncated)
|
||||
|
||||
|
||||
def delete_session(serial: str) -> bool:
|
||||
path = _path_for(serial)
|
||||
if not os.path.isfile(path):
|
||||
return False
|
||||
try:
|
||||
os.unlink(path)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
# Best-effort sidecar cleanup.
|
||||
try:
|
||||
os.unlink(_log_path_for(serial))
|
||||
except OSError:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def delete_all_sessions() -> int:
|
||||
"""Wipe every per-bay JSON in IMAGING_DIR. Returns count removed."""
|
||||
"""Wipe every per-bay JSON + sidecar .log in IMAGING_DIR. Returns count
|
||||
of JSON files removed."""
|
||||
_ensure_dir()
|
||||
removed = 0
|
||||
for fn in os.listdir(config.IMAGING_DIR):
|
||||
if not fn.endswith(".json"):
|
||||
continue
|
||||
try:
|
||||
os.unlink(os.path.join(config.IMAGING_DIR, fn))
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
if fn.endswith(".json") or fn.endswith(".log"):
|
||||
try:
|
||||
os.unlink(os.path.join(config.IMAGING_DIR, fn))
|
||||
except OSError:
|
||||
continue
|
||||
if fn.endswith(".json"):
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
Reference in New Issue
Block a user