diff --git a/plugins/endpoints/api/__init__.py b/plugins/endpoints/api/__init__.py index 53667058..608ce97c 100644 --- a/plugins/endpoints/api/__init__.py +++ b/plugins/endpoints/api/__init__.py @@ -5,6 +5,8 @@ from .routes import endpoints_bp from . import adoption # noqa: F401 # Attaches the site agent generator packaging routes to endpoints_bp. from . import siteagent # noqa: F401 +# Attaches the agent binary upload/retention routes to endpoints_bp. +from . import agentupload # noqa: F401 # Attaches the monitor catalog CRUD and resolve routes to endpoints_bp. from . import monitors # noqa: F401 diff --git a/plugins/endpoints/api/agentstore.py b/plugins/endpoints/api/agentstore.py new file mode 100644 index 00000000..d656dfef --- /dev/null +++ b/plugins/endpoints/api/agentstore.py @@ -0,0 +1,245 @@ +"""Versioned staging and retention for the endpoint agent binaries. + +The site-agent packager ships a prebuilt shopdbagent.exe and an optional +shopdb-endpoint-tray.exe. Originally each was a single staged file under +instance/siteagent/; this module adds per-version staging so an operator can +drop in a new build, keep the last few, and roll a push out from a known +version. + +Layout (all under the Flask instance directory): + + siteagent/ + shopdbagent.exe legacy flat path (pre-versioning) + shopdb-endpoint-tray.exe legacy flat path (pre-versioning) + agent/ + manifest.json {current, versions:[{version,uploadedat,size}]} + 1.6.3/shopdbagent.exe + 1.6.2/shopdbagent.exe + tray/ + manifest.json + 1.0.0/shopdb-endpoint-tray.exe + +Retention: the RETAIN most recently uploaded versions per binary are kept; the +current version is never pruned even if it falls outside that window. The legacy +flat file is treated as an unmanaged fallback: the packager still ships it when +no versioned build exists, so an install that predates versioning keeps working. + +This module holds NO routes and NO permission checks - it is pure staging logic +so it can be unit-tested without a request context and reused by both the upload +routes and the packager. Callers own authorization. +""" + +import io +import json +import os +import re +from datetime import datetime + +from flask import current_app + +# How many versions to retain per binary. The current version is exempt from +# pruning, so the effective floor is max(RETAIN, 1-including-current). +RETAIN = 3 + +# Per-binary staging spec. 'exe' is the canonical filename shipped in the ZIP and +# stored inside each version directory; 'legacy' is the flat pre-versioning path +# under siteagent/ kept as a fallback source. +BINARY_SPECS = { + 'agent': {'exe': 'shopdbagent.exe', 'legacy': 'shopdbagent.exe'}, + 'tray': {'exe': 'shopdb-endpoint-tray.exe', 'legacy': 'shopdb-endpoint-tray.exe'}, +} + +# A dotted numeric version, 1 to 4 parts (matches .NET assembly versions like +# 1.6.3 or 1.6.3.0). Deliberately strict so a typo or an arbitrary label cannot +# become a directory name. +_VERSION_RE = re.compile(r'^\d+(\.\d+){0,3}$') + +# Windows PE magic. Uploaded agent binaries must be real PE executables; a quick +# header check rejects an operator who uploads the wrong file (a ZIP, a script) +# before it is ever staged or pushed. +_PE_MAGIC = b'MZ' + + +def is_valid_binary_key(binkey): + """Whether binkey names a managed binary (agent or tray).""" + return binkey in BINARY_SPECS + + +def is_valid_version(version): + """Whether version is a strict dotted-numeric string safe to use as a dir.""" + return bool(version) and bool(_VERSION_RE.match(version)) + + +def looks_like_pe(data): + """Whether data begins with the Windows PE 'MZ' magic.""" + return len(data) >= 2 and data[:2] == _PE_MAGIC + + +def _siteagent_dir(): + return os.path.join(current_app.instance_path, 'siteagent') + + +def _binary_dir(binkey): + return os.path.join(_siteagent_dir(), binkey) + + +def _manifest_path(binkey): + return os.path.join(_binary_dir(binkey), 'manifest.json') + + +def _legacy_path(binkey): + return os.path.join(_siteagent_dir(), BINARY_SPECS[binkey]['legacy']) + + +def _version_exe_path(binkey, version): + return os.path.join(_binary_dir(binkey), version, BINARY_SPECS[binkey]['exe']) + + +def _read_manifest(binkey): + """Return the manifest dict, or an empty skeleton when none is written yet.""" + path = _manifest_path(binkey) + try: + with open(path, 'r', encoding='utf-8') as handle: + data = json.load(handle) + except (OSError, ValueError): + return {'current': None, 'versions': []} + if not isinstance(data, dict): + return {'current': None, 'versions': []} + data.setdefault('current', None) + data.setdefault('versions', []) + return data + + +def _write_manifest(binkey, manifest): + os.makedirs(_binary_dir(binkey), exist_ok=True) + with open(_manifest_path(binkey), 'w', encoding='utf-8') as handle: + json.dump(manifest, handle, indent=2) + handle.write('\n') + + +def list_versions(binkey): + """Staged versions for a binary, newest upload first. + + Each item is {version, uploadedat, size, iscurrent}. Reads the manifest as + the source of truth for order and currency; a version whose exe is missing + on disk is dropped so the caller never offers a broken push. + """ + manifest = _read_manifest(binkey) + current = manifest.get('current') + out = [] + for entry in manifest.get('versions', []): + version = entry.get('version') + if not version or not os.path.isfile(_version_exe_path(binkey, version)): + continue + out.append({ + 'version': version, + 'uploadedat': entry.get('uploadedat'), + 'size': entry.get('size'), + 'iscurrent': version == current, + }) + return out + + +def current_version(binkey): + """The current version string, or None when only a legacy flat file exists.""" + manifest = _read_manifest(binkey) + current = manifest.get('current') + if current and os.path.isfile(_version_exe_path(binkey, current)): + return current + return None + + +def current_path(binkey): + """Absolute path to the exe the packager should ship, or None. + + Prefers the current managed version; falls back to the legacy flat file so a + site that staged a binary before versioning still packages it. + """ + version = current_version(binkey) + if version: + return _version_exe_path(binkey, version) + legacy = _legacy_path(binkey) + if os.path.isfile(legacy): + return legacy + return None + + +def _prune(binkey, manifest): + """Drop versions past the retention window, never removing the current one.""" + current = manifest.get('current') + versions = manifest.get('versions', []) + keep = [] + removed = 0 + for index, entry in enumerate(versions): + version = entry.get('version') + # Keep the RETAIN newest (list is newest-first) plus the current always. + if index < RETAIN or version == current: + keep.append(entry) + continue + # Beyond the window and not current: remove its directory. + _remove_version_dir(binkey, version) + removed += 1 + manifest['versions'] = keep + return removed + + +def _remove_version_dir(binkey, version): + directory = os.path.join(_binary_dir(binkey), version) + try: + exe = _version_exe_path(binkey, version) + if os.path.isfile(exe): + os.remove(exe) + if os.path.isdir(directory) and not os.listdir(directory): + os.rmdir(directory) + except OSError: + # Best effort: a locked or already-gone file must not fail the upload + # that triggered pruning. The manifest no longer references it either way. + pass + + +def store_version(binkey, version, data): + """Stage an uploaded binary as a new version and make it current. + + Writes data to siteagent///, records it newest-first in + the manifest (de-duping a re-upload of the same version in place), marks it + current, and prunes older versions past RETAIN. Returns the fresh manifest + view {current, versions:[...]}. Caller must have validated binkey, version, + and that data is a PE binary. + """ + os.makedirs(os.path.join(_binary_dir(binkey), version), exist_ok=True) + with open(_version_exe_path(binkey, version), 'wb') as handle: + handle.write(data) + + manifest = _read_manifest(binkey) + versions = [entry for entry in manifest.get('versions', []) + if entry.get('version') != version] + versions.insert(0, { + 'version': version, + 'uploadedat': datetime.utcnow().isoformat() + 'Z', + 'size': len(data), + }) + manifest['versions'] = versions + manifest['current'] = version + _prune(binkey, manifest) + _write_manifest(binkey, manifest) + return { + 'current': manifest['current'], + 'versions': list_versions(binkey), + } + + +def set_current(binkey, version): + """Point 'current' at an already-staged version. Returns True on success. + + Used to roll back to a retained version. Refuses a version that is not + staged on disk so current never dangles. + """ + if not os.path.isfile(_version_exe_path(binkey, version)): + return False + manifest = _read_manifest(binkey) + if not any(entry.get('version') == version + for entry in manifest.get('versions', [])): + return False + manifest['current'] = version + _write_manifest(binkey, manifest) + return True diff --git a/plugins/endpoints/api/agentupload.py b/plugins/endpoints/api/agentupload.py new file mode 100644 index 00000000..c180a0a1 --- /dev/null +++ b/plugins/endpoints/api/agentupload.py @@ -0,0 +1,126 @@ +"""Agent binary upload and retention routes. + +Lets an operator drop in a new shopdbagent.exe or shopdb-endpoint-tray.exe from +the Endpoints page, keep the last few versions, and roll the current pointer +back to a retained build. The actual staging/retention logic lives in +agentstore.py; this module is only the HTTP surface and authorization. + +Authorization boundary (deliberate): + - Uploading, listing, and selecting the current staged version is + endpoints.agentpackage - the same grant that already lets someone build and + download the site-agent ZIP. Staging a binary only changes what a FUTURE + download contains; it does not touch any running endpoint. + - Pushing that binary to live endpoints (agent replacement) is a separate, + heavier grant, endpoints.agentupdate, and is NOT wired here. It is a distinct + command family from application deploys (endpoints.deploy) so app-push rights + never imply SYSTEM-level agent replacement. That push pipe is a later slice. + +Uploads are capped and PE-checked so a wrong or hostile file cannot be staged. +""" + +from flask import request + +from shopdb.api import (success_response, error_response, ErrorCodes, + require_permission) + +from .routes import endpoints_bp +from . import agentstore + +# Hard cap on an uploaded agent binary. The real agent is a few MB; this is a +# generous ceiling that still refuses an operator who points at a giant wrong +# file before it is buffered fully into memory-staged storage. +_MAX_UPLOAD_BYTES = 64 * 1024 * 1024 + + +@endpoints_bp.route('/siteagent/binaries/', methods=['GET']) +@require_permission('endpoints.agentpackage') +def list_agent_binaries(binkey): + """List staged versions for a binary (agent or tray), newest first.""" + if not agentstore.is_valid_binary_key(binkey): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'binkey must be one of: agent, tray', + http_code=400) + return success_response({ + 'binary': binkey, + 'current': agentstore.current_version(binkey), + 'versions': agentstore.list_versions(binkey), + }) + + +@endpoints_bp.route('/siteagent/binaries/', methods=['POST']) +@require_permission('endpoints.agentpackage') +def upload_agent_binary(binkey): + """Stage an uploaded binary as a new version and make it current. + + Expects multipart/form-data with a 'file' upload and a 'version' field. The + version must be a strict dotted-numeric string (for example 1.6.3) so it is + safe as a directory name and orderable. The upload must be a Windows PE + executable. Older versions past the retention window are pruned, but never + the current one. + """ + version = (request.form.get('version') or '').strip() + if not agentstore.is_valid_binary_key(binkey): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'binkey must be one of: agent, tray', + http_code=400) + if not agentstore.is_valid_version(version): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'version must be a dotted-numeric string such as 1.6.3', + http_code=400) + + upload = request.files.get('file') + if upload is None or not upload.filename: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'a file upload is required', + http_code=400) + + data = upload.read(_MAX_UPLOAD_BYTES + 1) + if len(data) > _MAX_UPLOAD_BYTES: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'uploaded binary exceeds the size limit', + http_code=400) + if not agentstore.looks_like_pe(data): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'uploaded file is not a Windows executable', + http_code=400) + + result = agentstore.store_version(binkey, version, data) + return success_response({ + 'binary': binkey, + 'current': result['current'], + 'versions': result['versions'], + }) + + +@endpoints_bp.route('/siteagent/binaries//current', methods=['PUT']) +@require_permission('endpoints.agentpackage') +def set_current_agent_binary(binkey): + """Point 'current' at an already-staged version (roll back or forward).""" + if not agentstore.is_valid_binary_key(binkey): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'binkey must be one of: agent, tray', + http_code=400) + source = request.get_json(silent=True) or {} + version = (source.get('version') or '').strip() + if not agentstore.is_valid_version(version): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'version must be a dotted-numeric string such as 1.6.3', + http_code=400) + if not agentstore.set_current(binkey, version): + return error_response( + ErrorCodes.NOT_FOUND, + 'that version is not staged', + http_code=404) + return success_response({ + 'binary': binkey, + 'current': agentstore.current_version(binkey), + 'versions': agentstore.list_versions(binkey), + }) diff --git a/plugins/endpoints/api/siteagent.py b/plugins/endpoints/api/siteagent.py index e261111b..216c1121 100644 --- a/plugins/endpoints/api/siteagent.py +++ b/plugins/endpoints/api/siteagent.py @@ -35,6 +35,7 @@ from shopdb.api import (success_response, error_response, ErrorCodes, require_permission) from .routes import endpoints_bp +from . import agentstore # Prebuilt, staged agent binaries. Built out of band and dropped under # instance/siteagent/ (mirrors the adoption tool's STAGED_EXE_RELPATH model). @@ -446,13 +447,21 @@ else {{ def _siteagent_status_payload(): - """Whether a prebuilt agent binary is staged for packaging.""" - agent_staged = os.path.isfile(_staged_path(STAGED_AGENT_RELPATH)) - tray_staged = os.path.isfile(_staged_path(STAGED_TRAY_RELPATH)) + """Whether a prebuilt agent binary is staged for packaging. + + Reports both the legacy flat-file staging and the versioned store so the UI + can show current versions and an expandable retained-version history. + """ + agent_staged = agentstore.current_path('agent') is not None + tray_staged = agentstore.current_path('tray') is not None return { 'agentstaged': agent_staged, 'traystaged': tray_staged, 'certavailable': _server_cert_path() is not None, + 'agentcurrent': agentstore.current_version('agent'), + 'traycurrent': agentstore.current_version('tray'), + 'agentversions': agentstore.list_versions('agent'), + 'trayversions': agentstore.list_versions('tray'), } @@ -545,11 +554,11 @@ def package_siteagent(): if os.path.isfile(UNINSTALLER_PATH): archive.write(UNINSTALLER_PATH, 'uninstall-shopdbagent.ps1') - agent = _staged_path(STAGED_AGENT_RELPATH) - if os.path.isfile(agent): + agent = agentstore.current_path('agent') + if agent and os.path.isfile(agent): archive.write(agent, 'shopdbagent.exe') - tray = _staged_path(STAGED_TRAY_RELPATH) - if os.path.isfile(tray): + tray = agentstore.current_path('tray') + if tray and os.path.isfile(tray): archive.write(tray, 'shopdb-endpoint-tray.exe') if cert_pem: archive.writestr('server.crt', cert_pem) diff --git a/plugins/endpoints/plugin.py b/plugins/endpoints/plugin.py index 4557ec31..076aac18 100644 --- a/plugins/endpoints/plugin.py +++ b/plugins/endpoints/plugin.py @@ -179,6 +179,19 @@ class EndpointsPlugin(BasePlugin): ('endpoints.agentpackage', 'Generate and download an agent or adoption-tool package ' '(contains a live collector key)', 'endpoints'), + # Pushing an agent REPLACEMENT to running endpoints is a distinct, + # heavier job than either packaging (endpoints.agentpackage) or + # pushing an ordinary application installer (endpoints.deploy). + # Kept separate on purpose: the agent runs as SYSTEM and is the + # command channel itself, so replacing it is self-modifying the + # control plane. Folding it into endpoints.deploy would let anyone + # who can push an app silently swap the agent; folding it into + # endpoints.agentpackage would let a packager who only mints ZIPs + # also force a live fleet-wide swap. A future Operator role can + # hold upload without this, or this without upload. + ('endpoints.agentupdate', + 'Push agent/tray binary updates to running endpoints ' + '(replaces the SYSTEM agent)', 'endpoints'), # Collector service scope for the endpoint agent. A managed token # scoped to ONLY this authorizes the endpoints collector and nothing # else, so rotating/revoking it stops endpoint-agent traffic without