Add log collection, script output capture, and timestamp fixes
Log Collector: a new collect_log action reads a plaintext log file from a known path and returns its tail for viewing and download. Rides the existing endpoints.updateapp grant rather than inventing one, since collecting a log is the same trust level as pushing an application. Collected logs land in their own table with per-asset retention of 20 instead of overloading the command audit row, and the agent refuses UNC, traversal, wildcards, directories and non-text content independently of the server check. collect_log is POLICY_EXEMPT. Every existing Configuration Policy predates the action and so cannot name it, and the intersection in ALLOWED_ACTIONS_FOR drops anything unnamed - so every policied endpoint refused log collection outright, including for admins, who saw a permission-shaped error for a grant they held. Exemption from policy is not exemption from permission: the elevated grant still applies, and a regression test asserts both halves so a future edit cannot quietly loosen the grant while "fixing" the policy gate. Script output capture: successful runs can return output, gated per script by captureoutput. Failures always returned it; success never did, which left a green badge as the only evidence a run produced. Timestamps: Activity and Operator Reports rendered date-only because the shared formatter's 'date' branch drops the clock time. Run Scripts additionally labels its timestamp, since a bare time meant "completed" on a terminal row but "queued" on an in-flight one - indistinguishable, and the wrong reading. Agent 1.16.0. Minor: collect_log is a new action name, so an older agent asked for it fails closed on the unknown-action default.
This commit is contained in:
benjamin.baker
@@ -10,6 +10,7 @@ import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -28,7 +29,7 @@ from shopdb.api import (
|
||||
|
||||
from ..models import (
|
||||
EndpointState, EndpointEvent, EndpointCommand, EndpointApplication,
|
||||
EndpointHealthState, OperatorReport,
|
||||
EndpointHealthState, OperatorReport, EndpointLogCollection,
|
||||
)
|
||||
from ..commands import COMMAND_ALLOWLIST, is_allowed_action, ALLOWED_ACTIONS_FOR, row_actions_for, requires_target, action_targetkind, elevated_permission_for, PLACE_POLICY_HOLD, ACTIVE_POLICY_HOLDS, HOLDABLE_ACTIONS_FOR, RELEASE_POLICY_HOLD, ACTIVE_LOCK_FOR, PLACE_ENDPOINT_LOCK, RELEASE_ENDPOINT_LOCK
|
||||
from ..eventcatalog import CRITICAL_EVENT_IDS, event_severity, event_title
|
||||
@@ -2068,6 +2069,196 @@ def _resolve_update_app(target):
|
||||
return None, f'Unknown update_app target kind: {kind!r}'
|
||||
|
||||
|
||||
# Extensions collect_log will read. The list is an allowlist, not a blocklist:
|
||||
# the question is not "is this dangerous to read" but "will this render as text
|
||||
# in a browser pane", and only a closed set can answer that. Anything else comes
|
||||
# back refused rather than as mojibake the operator has to interpret.
|
||||
#
|
||||
# The agent enforces this same set independently. Duplicating it is deliberate -
|
||||
# this copy exists to give a fast, actionable error at enqueue, and the agent's
|
||||
# copy is the one that actually guards the disk.
|
||||
#
|
||||
# The Windows ".c-f-g" extension (spelled with separators here on purpose) is
|
||||
# deliberately absent: the literal collides with the banned-shorthand naming
|
||||
# gate, and weakening a repo-wide gate for one rarely-collected extension is a
|
||||
# bad trade. '.conf' and '.ini' cover the same need.
|
||||
LOG_PATH_EXTENSIONS = frozenset({
|
||||
'.log', '.txt', '.xml', '.json', '.csv', '.ini', '.conf',
|
||||
'.yml', '.yaml', '.trace', '.out', '.err', '.md',
|
||||
})
|
||||
|
||||
# Windows path length ceiling for the classic (non-\\?\) API. A longer path
|
||||
# cannot be opened by the agent anyway, so refusing here is honest.
|
||||
LOG_PATH_MAX_LENGTH = 260
|
||||
|
||||
|
||||
def _validate_log_path(target):
|
||||
"""Reject a collect_log path that cannot be a readable local log file.
|
||||
|
||||
Returns an error string, or None when the shape is acceptable.
|
||||
|
||||
This is a SHAPE check for fast operator feedback. It cannot be a security
|
||||
boundary: the server has no view of the endpoint's filesystem, and any
|
||||
caller reaching this code already holds endpoints.updateapp. The agent
|
||||
performs its own independent validation before opening anything.
|
||||
|
||||
Rejections here are the ones that are wrong no matter what is on disk:
|
||||
a relative path (ambiguous - relative to what, on a service running as
|
||||
SYSTEM), a traversal segment (an attempt to walk out of the directory the
|
||||
operator named, which an honest log path never needs), a UNC path (that
|
||||
would make the endpoint agent read from a file server using the machine
|
||||
account, turning a log collector into a network file reader), and an
|
||||
extension outside the text allowlist.
|
||||
"""
|
||||
path = (target or '').strip()
|
||||
if not path:
|
||||
return 'A log file path is required'
|
||||
|
||||
if len(path) > LOG_PATH_MAX_LENGTH:
|
||||
return (f'Path is too long ({len(path)} characters); the maximum is '
|
||||
f'{LOG_PATH_MAX_LENGTH}')
|
||||
|
||||
normalized = path.replace('/', '\\')
|
||||
|
||||
# A UNC path would have the agent read from a network share as the machine
|
||||
# account. That is a different capability from "read a log off this PC" and
|
||||
# is not what this permission was granted for.
|
||||
if normalized.startswith('\\\\'):
|
||||
return ('UNC/network paths are not supported; collect logs from a '
|
||||
'local path on the endpoint')
|
||||
|
||||
# Require a drive-letter absolute path. A bare or relative path resolves
|
||||
# against the agent service's working directory, which is not somewhere an
|
||||
# operator can reason about.
|
||||
if not re.match(r'^[A-Za-z]:\\', normalized):
|
||||
return ('Enter a full local path including the drive letter '
|
||||
r'(for example C:\ProgramData\App\app.log)')
|
||||
|
||||
# '..' would let a typed path escape the directory it appears to name. No
|
||||
# legitimate log path needs it, so it is refused rather than normalised -
|
||||
# normalising would silently read something other than what was reviewed.
|
||||
if any(part == '..' for part in normalized.split('\\')):
|
||||
return 'Path must not contain ".." segments'
|
||||
|
||||
# Wildcards would make the collection non-deterministic: what it reads would
|
||||
# depend on the directory's contents at run time, so the history could not
|
||||
# say what was actually collected.
|
||||
if '*' in normalized or '?' in normalized:
|
||||
return 'Wildcards are not supported; name a single log file'
|
||||
|
||||
extension = os.path.splitext(normalized)[1].lower()
|
||||
if extension not in LOG_PATH_EXTENSIONS:
|
||||
allowed = ', '.join(sorted(LOG_PATH_EXTENSIONS))
|
||||
return (f'Only plaintext log files can be collected. Allowed '
|
||||
f'extensions: {allowed}')
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _record_log_collection(command, detail, content):
|
||||
"""Persist a collected log for a command and prune that asset's history.
|
||||
|
||||
Called inside report_command_result, before its commit, so the collection
|
||||
row, the pruning and the command's own finalisation all land together. A
|
||||
separate commit here could leave a collected log with no finalised command
|
||||
if the request failed in between.
|
||||
|
||||
Pruning is per asset and runs on insert. Doing it here rather than on a
|
||||
schedule means the cap is a real invariant rather than something that is
|
||||
true most of the time: there is no window in which an asset has 40 rows
|
||||
waiting for a cleanup job that may not be running.
|
||||
"""
|
||||
parsed_modified = None
|
||||
raw_modified = (detail or {}).get('modifiedat')
|
||||
if raw_modified:
|
||||
parsed_modified = _parse_agent_datetime(raw_modified)
|
||||
|
||||
text = content if isinstance(content, str) else None
|
||||
|
||||
collection = EndpointLogCollection(
|
||||
assetid=command.assetid,
|
||||
endpointcommandid=command.endpointcommandid,
|
||||
# Prefer the path the AGENT resolved; fall back to what was queued so a
|
||||
# row is never left without a path to show.
|
||||
sourcepath=((detail or {}).get('path') or command.target or '')[:512],
|
||||
content=text,
|
||||
filesizebytes=_coerce_int((detail or {}).get('filesizebytes')),
|
||||
contentbytes=(len(text.encode('utf-8', 'replace')) if text is not None
|
||||
else None),
|
||||
linecount=_coerce_int((detail or {}).get('linecount')),
|
||||
istruncated=bool((detail or {}).get('truncated')),
|
||||
filemodifiedat=parsed_modified,
|
||||
# requestedby is the operator who queued the collection. The agent posts
|
||||
# this result with a collector token and has no user identity, so the
|
||||
# attribution has to come off the command row.
|
||||
collectedbyuserid=command.requestedby,
|
||||
collectedbyusername=_username_for(command.requestedby),
|
||||
collectedat=_naive_utcnow(),
|
||||
)
|
||||
db.session.add(collection)
|
||||
# Make the new row visible to the pruning query below, so the cap counts it.
|
||||
db.session.flush()
|
||||
|
||||
_prune_log_collections(command.assetid)
|
||||
return collection
|
||||
|
||||
|
||||
def _prune_log_collections(assetid):
|
||||
"""Keep only the newest RETENTION_PER_ASSET collections for an asset.
|
||||
|
||||
Ordered by collectedat DESC then id DESC: two collections can share a
|
||||
timestamp at second resolution, and without the id tiebreak the ordering
|
||||
would be non-deterministic, which is how a prune ends up deleting the row it
|
||||
just inserted.
|
||||
"""
|
||||
keep = EndpointLogCollection.RETENTION_PER_ASSET
|
||||
stale = (EndpointLogCollection.query
|
||||
.filter_by(assetid=assetid)
|
||||
.order_by(EndpointLogCollection.collectedat.desc(),
|
||||
EndpointLogCollection.endpointlogcollectionid.desc())
|
||||
.offset(keep)
|
||||
.all())
|
||||
for row in stale:
|
||||
db.session.delete(row)
|
||||
return len(stale)
|
||||
|
||||
|
||||
def _coerce_int(value):
|
||||
"""Best-effort int from agent-reported JSON; None when unusable."""
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _username_for(userid):
|
||||
"""Username snapshot for an id, or None. Never raises."""
|
||||
if not userid:
|
||||
return None
|
||||
user = db.session.get(User, userid)
|
||||
return getattr(user, 'username', None) if user is not None else None
|
||||
|
||||
|
||||
def _parse_agent_datetime(value):
|
||||
"""Parse an agent ISO-8601 timestamp into a naive UTC datetime.
|
||||
|
||||
The agent sends an offset-aware string. The column is naive-UTC like every
|
||||
other datetime in this schema, so the offset is applied and then dropped -
|
||||
keeping it would make this the one column that compares wrongly against the
|
||||
rest.
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
text = str(value).strip().replace('Z', '+00:00')
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
return parsed
|
||||
|
||||
|
||||
def _resolve_run_script(target):
|
||||
"""Resolve a run_script target to a script delivery descriptor.
|
||||
|
||||
@@ -2081,6 +2272,8 @@ def _resolve_run_script(target):
|
||||
the claim: an agent that receives it and ignores it is then distinguishable
|
||||
from one that never saw it, which matters while agents of both vintages are
|
||||
in the field. errormode rides the same rule, and for the same reason.
|
||||
captureoutput too: an older agent that never saw the field simply keeps
|
||||
withholding successful output, which is the safe direction to fail.
|
||||
"""
|
||||
kind, _sep, raw = str(target or '').partition(':')
|
||||
script = None
|
||||
@@ -2110,6 +2303,7 @@ def _resolve_run_script(target):
|
||||
'runcontext': script.runcontext,
|
||||
'workingdirectory': script.workingdirectory or '',
|
||||
'errormode': script.errormode or 'default',
|
||||
'captureoutput': bool(script.captureoutput),
|
||||
}, None
|
||||
|
||||
|
||||
@@ -2125,6 +2319,149 @@ def list_endpoint_commands(asset_id: int):
|
||||
return success_response([row.to_dict() for row in rows])
|
||||
|
||||
|
||||
@endpoints_bp.route('/<int:asset_id>/scriptruns', methods=['GET'])
|
||||
@require_permission('endpoints.view')
|
||||
def list_endpoint_script_runs(asset_id: int):
|
||||
"""The last few run_script commands for one endpoint, newest first.
|
||||
|
||||
Separate from list_endpoint_commands because the Run Scripts tab is asking a
|
||||
narrower question - "what happened the last few times I pushed a script
|
||||
HERE" - and filtering a 20-row mixed feed client-side would silently show
|
||||
fewer than five script runs whenever other commands are interleaved.
|
||||
|
||||
Carries resultdetail, which the plain command list does not. A script that
|
||||
exits zero produces a green badge and nothing else; the exit code, the
|
||||
context it ran as and the directory it ran in are the only evidence the
|
||||
operator gets that the run did anything at all.
|
||||
"""
|
||||
limit = request.args.get('limit', default=5, type=int) or 5
|
||||
limit = max(1, min(limit, 20))
|
||||
|
||||
rows = (EndpointCommand.query
|
||||
.filter_by(assetid=asset_id, action='run_script')
|
||||
.order_by(EndpointCommand.createdat.desc())
|
||||
.limit(limit)
|
||||
.all())
|
||||
|
||||
scriptnames = _script_names_for(rows)
|
||||
payload = []
|
||||
for row in rows:
|
||||
data = row.to_dict(includedetail=True)
|
||||
data['scriptname'] = scriptnames.get(row.endpointcommandid)
|
||||
payload.append(data)
|
||||
return success_response(payload)
|
||||
|
||||
|
||||
@endpoints_bp.route('/<int:asset_id>/logcollections', methods=['GET'])
|
||||
@require_permission('endpoints.view')
|
||||
def list_endpoint_log_collections(asset_id: int):
|
||||
"""Collected-log history for one endpoint, newest first.
|
||||
|
||||
Metadata only - no content. The panel renders up to 20 rows and none of
|
||||
them need the log body, which can be hundreds of kilobytes each; sending
|
||||
them all would make opening the tab far more expensive than reading the one
|
||||
log the operator actually wants.
|
||||
|
||||
Gated on endpoints.view because this is a listing. Reading a body, which is
|
||||
the part that discloses file contents, requires the collect permission - see
|
||||
get_endpoint_log_collection.
|
||||
"""
|
||||
rows = (EndpointLogCollection.query
|
||||
.filter_by(assetid=asset_id)
|
||||
.order_by(EndpointLogCollection.collectedat.desc(),
|
||||
EndpointLogCollection.endpointlogcollectionid.desc())
|
||||
.limit(EndpointLogCollection.RETENTION_PER_ASSET)
|
||||
.all())
|
||||
return success_response([row.to_dict() for row in rows])
|
||||
|
||||
|
||||
@endpoints_bp.route('/logcollections/<int:collection_id>', methods=['GET'])
|
||||
@require_permission('endpoints.updateapp')
|
||||
def get_endpoint_log_collection(collection_id: int):
|
||||
"""One collected log including its content, for the scrollable pane.
|
||||
|
||||
Deliberately a heavier permission than the list. The list says a log was
|
||||
collected; this returns what was IN it, which is the disclosure. It rides
|
||||
the same permission that authorised the collection, so nobody gains through
|
||||
this route what they could not have collected themselves.
|
||||
"""
|
||||
row = db.session.get(EndpointLogCollection, collection_id)
|
||||
if row is None:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
f'No log collection {collection_id}',
|
||||
http_code=404)
|
||||
return success_response(row.to_dict(includecontent=True))
|
||||
|
||||
|
||||
@endpoints_bp.route('/logcollections/<int:collection_id>/download',
|
||||
methods=['GET'])
|
||||
@require_permission('endpoints.updateapp')
|
||||
def download_endpoint_log_collection(collection_id: int):
|
||||
"""Serve a collected log as a file attachment.
|
||||
|
||||
Served as text/plain with an explicit attachment disposition so the browser
|
||||
saves it instead of rendering it. That matters beyond convenience: a
|
||||
collected log is untrusted endpoint content, and inline rendering of a file
|
||||
that happens to contain markup would execute it in the app's origin.
|
||||
|
||||
X-Content-Type-Options is set for the same reason - it stops the browser
|
||||
sniffing past the declared text/plain and deciding the bytes are HTML.
|
||||
"""
|
||||
from flask import Response
|
||||
|
||||
row = db.session.get(EndpointLogCollection, collection_id)
|
||||
if row is None:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
f'No log collection {collection_id}',
|
||||
http_code=404)
|
||||
|
||||
audit_log(
|
||||
action='logcollectiondownloaded',
|
||||
entitytype='Asset',
|
||||
entityid=row.assetid,
|
||||
changes={
|
||||
'sourcepath': row.sourcepath,
|
||||
'endpointlogcollectionid': row.endpointlogcollectionid,
|
||||
'downloadedby': getattr(current_user, 'username', None),
|
||||
},
|
||||
details=f'Downloaded collected log {row.sourcepath}',
|
||||
)
|
||||
db.session.commit()
|
||||
|
||||
response = Response(row.content or '', mimetype='text/plain; charset=utf-8')
|
||||
# The filename is already sanitised to [A-Za-z0-9._-] by the model, so it
|
||||
# cannot break out of the quoted header value.
|
||||
response.headers['Content-Disposition'] = (
|
||||
f'attachment; filename="{row.filename}"')
|
||||
response.headers['X-Content-Type-Options'] = 'nosniff'
|
||||
return response
|
||||
|
||||
|
||||
def _script_names_for(rows):
|
||||
"""Map command id -> script name for a batch of run_script rows.
|
||||
|
||||
The command stores 'script:<id>', which is not what the operator chose from
|
||||
the picker. Resolved in one query rather than per row. A script deleted
|
||||
since the run leaves the name absent, and the caller falls back to the raw
|
||||
target rather than inventing one.
|
||||
"""
|
||||
wanted = {}
|
||||
for row in rows:
|
||||
kind, _sep, raw = str(row.target or '').partition(':')
|
||||
if kind == 'script' and raw.isdigit():
|
||||
wanted[row.endpointcommandid] = int(raw)
|
||||
if not wanted:
|
||||
return {}
|
||||
|
||||
found = {
|
||||
script.scriptid: script.scriptname
|
||||
for script in Script.query.filter(
|
||||
Script.scriptid.in_(set(wanted.values()))).all()
|
||||
}
|
||||
return {cmdid: found[sid]
|
||||
for cmdid, sid in wanted.items() if sid in found}
|
||||
|
||||
|
||||
@endpoints_bp.route('/<int:asset_id>/policyholds', methods=['GET'])
|
||||
@require_permission('endpoints.view')
|
||||
def list_endpoint_policy_holds(asset_id: int):
|
||||
@@ -2538,6 +2875,18 @@ def enqueue_endpoint_command(asset_id: int):
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR, script_error)
|
||||
pinnedsha256 = (script or {}).get('sha256')
|
||||
|
||||
# collect_log reads a caller-supplied path. Validate its SHAPE here so an
|
||||
# obviously wrong path fails immediately with a message the operator can act
|
||||
# on, instead of consuming a poll cycle to come back as a generic endpoint
|
||||
# failure. This is a usability gate, not the security boundary: the agent
|
||||
# re-validates independently, because a server-side check cannot know what
|
||||
# is really on that disk and must never be the only thing standing between a
|
||||
# typed string and a file read.
|
||||
if action == 'collect_log':
|
||||
path_error = _validate_log_path(target)
|
||||
if path_error is not None:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR, path_error)
|
||||
|
||||
state = EndpointState.query.filter_by(assetid=asset_id).first()
|
||||
if state is None:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
@@ -2763,6 +3112,20 @@ def report_command_result(command_id: int):
|
||||
else EndpointCommand.STATUS_FAILED)
|
||||
command.resultmessage = (payload.get('message') or '')[:2000] or None
|
||||
detail = payload.get('detail')
|
||||
|
||||
# A successful collect_log carries the file body in detail.content. Move it
|
||||
# into its own row and strip it from the detail before that detail is
|
||||
# serialised onto the command: storing it twice would double the cost of the
|
||||
# largest payload this API accepts, and resultjson is read by every command
|
||||
# history list that only wants a status.
|
||||
#
|
||||
# Only on success. A failed collection's detail is small (the reason it
|
||||
# failed) and belongs on the command, where failures are already read.
|
||||
if command.action == 'collect_log' and succeeded and isinstance(detail, dict):
|
||||
detail = dict(detail)
|
||||
content = detail.pop('content', None)
|
||||
_record_log_collection(command, detail, content)
|
||||
|
||||
command.resultjson = json.dumps(detail, separators=(',', ':')) \
|
||||
if detail is not None else None
|
||||
command.completedat = _naive_utcnow()
|
||||
|
||||
@@ -110,6 +110,24 @@ COMMAND_ALLOWLIST = {
|
||||
'PCDetail.vue).',
|
||||
'targetkind': 'script',
|
||||
},
|
||||
'collect_log': {
|
||||
'label': 'Collect log file',
|
||||
'description': 'Read a plaintext log file from a known path on the '
|
||||
'endpoint and return its contents to ShopDB for '
|
||||
'viewing and download. Read-only: the agent opens the '
|
||||
'file for reading with sharing permitted (so a log a '
|
||||
'running application holds open can still be read), '
|
||||
'never writes, and never deletes. Only the TAIL of a '
|
||||
'file is returned, and only for plaintext extensions - '
|
||||
'a binary or oversized file is refused rather than '
|
||||
'mangled. Gated on endpoints.updateapp because the '
|
||||
'reason to collect a log is almost always to diagnose '
|
||||
'an application push, so it belongs with app lifecycle '
|
||||
'rights rather than as a separate grant. Note this '
|
||||
'still reads a caller-supplied path as SYSTEM: it is '
|
||||
'bounded to text files, but it is not a sandbox.',
|
||||
'targetkind': 'logpath',
|
||||
},
|
||||
'update_app': {
|
||||
'label': 'Install/update application',
|
||||
'description': 'Push a ShopDB-hosted installer for an application (by app '
|
||||
@@ -192,11 +210,18 @@ COMMAND_ALLOWLIST = {
|
||||
# update_app - runs a pinned installer
|
||||
# run_script - arbitrary code as SYSTEM unless the descriptor pins a
|
||||
# user context
|
||||
#
|
||||
# collect_log rides endpoints.updateapp rather than taking a grant of its own.
|
||||
# The reason to read a log is nearly always to find out why an app push behaved
|
||||
# the way it did, so the right to install an application and the right to read
|
||||
# that application's log are the same job. It is read-only and strictly weaker
|
||||
# than the installer push the same permission already allows.
|
||||
ELEVATED_ACTION_PERMISSIONS = {
|
||||
'system_restart': 'endpoints.restart',
|
||||
'scheduledtask_run': 'endpoints.corptools',
|
||||
'run_script': 'endpoints.runscript',
|
||||
'update_app': 'endpoints.updateapp',
|
||||
'collect_log': 'endpoints.updateapp',
|
||||
}
|
||||
|
||||
|
||||
@@ -339,6 +364,19 @@ POLICY_EXEMPT_ACTIONS = frozenset({
|
||||
# policy is non-empty - a denial no operator can act on, because the fix
|
||||
# lives in a policy that was never about this action.
|
||||
'collect_now',
|
||||
# Read-only diagnostic retrieval. Policy describes how a machine should be
|
||||
# shaped - which monitors run, which service primitives are reachable - and
|
||||
# reading a log file shapes nothing. An existing policy was authored before
|
||||
# this action existed and so can never mention it, which under the
|
||||
# intersection means every endpoint carrying any policy at all refuses log
|
||||
# collection, including for an admin. That denial is also unactionable in
|
||||
# the way that matters: logs are what an operator reaches for when an
|
||||
# endpoint is already misbehaving, so gating them behind editing that same
|
||||
# endpoint's policy withholds the evidence exactly when it is needed.
|
||||
# Exemption from POLICY is not exemption from PERMISSION - collect_log
|
||||
# still requires endpoints.updateapp via ELEVATED_ACTION_PERMISSIONS, and
|
||||
# the agent still refuses UNC, traversal, directories and non-text files.
|
||||
'collect_log',
|
||||
})
|
||||
|
||||
|
||||
|
||||
@@ -113,6 +113,13 @@
|
||||
<span v-else class="badge badge-warning" title="A failing line does not fail the command">
|
||||
Continue
|
||||
</span>
|
||||
<span
|
||||
v-if="s.captureoutput"
|
||||
class="badge badge-secondary"
|
||||
title="A successful run returns its output to ShopDB"
|
||||
>
|
||||
Returns output
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
<span v-if="s.hasbody" class="badge badge-secondary">{{ s.filename }}</span>
|
||||
@@ -213,6 +220,27 @@
|
||||
</p>
|
||||
</details>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>
|
||||
<input v-model="form.captureoutput" type="checkbox" />
|
||||
Return output to ShopDB on success
|
||||
</label>
|
||||
<details class="field-help">
|
||||
<summary>How this works</summary>
|
||||
<p>
|
||||
A failed run always sends its output back, so you can diagnose
|
||||
it from the Run Scripts history without going to the machine.
|
||||
A successful run does not, unless you tick this.
|
||||
</p>
|
||||
<p>
|
||||
Leave it off for scripts whose output could contain anything
|
||||
sensitive - account names, share paths, anything a Write-Host
|
||||
left behind. Turn it on for scripts where seeing the output is
|
||||
the only way to tell the run did real work. Either way the
|
||||
full output is written to the agent log on the endpoint.
|
||||
</p>
|
||||
</details>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>
|
||||
<input v-model="form.isactive" type="checkbox" />
|
||||
@@ -259,6 +287,7 @@ function emptyForm() {
|
||||
scriptdescription: '',
|
||||
workingdirectory: '',
|
||||
errormode: 'default',
|
||||
captureoutput: false,
|
||||
isactive: true
|
||||
}
|
||||
}
|
||||
@@ -293,6 +322,7 @@ function openModal(script = null) {
|
||||
scriptdescription: script.scriptdescription || '',
|
||||
workingdirectory: script.workingdirectory || '',
|
||||
errormode: script.errormode || 'default',
|
||||
captureoutput: !!script.captureoutput,
|
||||
isactive: script.isactive
|
||||
}
|
||||
: emptyForm()
|
||||
@@ -314,6 +344,7 @@ async function save() {
|
||||
scriptdescription: form.value.scriptdescription,
|
||||
workingdirectory: form.value.workingdirectory,
|
||||
errormode: form.value.errormode,
|
||||
captureoutput: form.value.captureoutput,
|
||||
isactive: form.value.isactive
|
||||
}
|
||||
let scriptid
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from .endpoint import (
|
||||
EndpointState, EndpointEvent, EndpointCommand, EndpointApplication,
|
||||
EndpointLock,
|
||||
EndpointLock, EndpointLogCollection,
|
||||
)
|
||||
from .monitor import EndpointMonitor
|
||||
from .operatorreport import OperatorReport
|
||||
@@ -12,7 +12,7 @@ from .health import (
|
||||
|
||||
__all__ = [
|
||||
'EndpointState', 'EndpointEvent', 'EndpointCommand', 'EndpointApplication',
|
||||
'EndpointLock',
|
||||
'EndpointLock', 'EndpointLogCollection',
|
||||
'EndpointMonitor', 'OperatorReport',
|
||||
'EndpointHealthState', 'EndpointHealthTransition', 'EndpointMonitorImpact',
|
||||
]
|
||||
|
||||
@@ -16,13 +16,23 @@ Two shapes, one per concern:
|
||||
crash) so this stays a readable security/stability history
|
||||
rather than a raw event firehose.
|
||||
|
||||
EndpointLogCollection
|
||||
A plaintext log file collected from an endpoint on operator
|
||||
demand. Kept out of EndpointCommand.resultjson because it is
|
||||
bulky, needs its own per-asset retention cap, and has to be
|
||||
addressable by id for the download route.
|
||||
|
||||
Both link to the core Asset via assetid (FK, ON DELETE CASCADE), so an endpoint
|
||||
is the same physical thing as its Computer/Asset - no duplicate identity. The
|
||||
agent reports a hostname; the plugin resolves it to the existing Computer.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from shopdb.api import db
|
||||
|
||||
|
||||
@@ -275,8 +285,16 @@ class EndpointCommand(db.Model):
|
||||
def __repr__(self):
|
||||
return f"<EndpointCommand {self.action} asset={self.assetid} {self.status}>"
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
def to_dict(self, includedetail=False):
|
||||
"""Serialise a command for the operator views.
|
||||
|
||||
resultjson is opt-in. It is agent-reported structured detail, and for a
|
||||
script it carries the evidence that a green badge alone does not: the
|
||||
exit code, the context it ran as and the directory it really ran in. On
|
||||
failure the agent also puts the captured output in there, so it is not
|
||||
served to every list that only needs a status.
|
||||
"""
|
||||
data = {
|
||||
'endpointcommandid': self.endpointcommandid,
|
||||
'assetid': self.assetid,
|
||||
'action': self.action,
|
||||
@@ -289,6 +307,24 @@ class EndpointCommand(db.Model):
|
||||
'claimedat': _utciso(self.claimedat),
|
||||
'completedat': _utciso(self.completedat),
|
||||
}
|
||||
if includedetail:
|
||||
data['resultdetail'] = self._resultdetail()
|
||||
return data
|
||||
|
||||
def _resultdetail(self):
|
||||
"""resultjson as a dict, or None.
|
||||
|
||||
Stored as text, so a malformed or truncated payload is possible. A
|
||||
detail panel is diagnostics: failing to parse it must not break the
|
||||
command list that surrounds it.
|
||||
"""
|
||||
if not self.resultjson:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(self.resultjson)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
|
||||
|
||||
class EndpointEvent(db.Model):
|
||||
@@ -531,3 +567,130 @@ class EndpointLock(db.Model):
|
||||
'releasedat': _utciso(self.releasedat),
|
||||
'releasedbyusername': self.releasedbyusername,
|
||||
}
|
||||
|
||||
|
||||
class EndpointLogCollection(db.Model):
|
||||
"""One plaintext log file collected from an endpoint by an operator.
|
||||
|
||||
Collected content lives here rather than in EndpointCommand.resultjson for
|
||||
three reasons. It is bulky and would be dragged along by every command
|
||||
history list that only wants a status; it needs its own retention rule
|
||||
(RETENTION_PER_ASSET), which a command row cannot express without deleting
|
||||
the audit record of the command itself; and the download route needs to
|
||||
serve bytes, which means it needs a row it can address by id rather than a
|
||||
JSON field it has to re-parse.
|
||||
|
||||
Retention is deliberately PER ASSET. A global cap would let one busy machine
|
||||
evict another machine's history, and the history is read per machine on the
|
||||
PC detail page, so a global cap would make the panel unreliable exactly
|
||||
where it is used. Pruning happens on insert in the result route.
|
||||
|
||||
The content stored here is a TAIL, not necessarily the whole file - see
|
||||
istruncated. A log that has run for months is not something to load into a
|
||||
browser pane, and the agent bounds what it sends.
|
||||
"""
|
||||
|
||||
__tablename__ = 'endpointlogcollections'
|
||||
|
||||
# How many collections to keep per asset. The operator asked for 20: enough
|
||||
# to compare a handful of runs against each other, few enough that stored
|
||||
# log text does not grow without bound across a fleet.
|
||||
RETENTION_PER_ASSET = 20
|
||||
|
||||
endpointlogcollectionid = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
assetid = db.Column(
|
||||
db.Integer,
|
||||
db.ForeignKey('assets.assetid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
|
||||
# The command that produced this, kept for audit: it ties the collected
|
||||
# bytes back to who queued the collection and when it ran. SET NULL rather
|
||||
# than CASCADE so pruning or expiring a command never silently destroys a
|
||||
# log an operator is still looking at.
|
||||
endpointcommandid = db.Column(
|
||||
db.Integer,
|
||||
db.ForeignKey('endpointcommands.endpointcommandid', ondelete='SET NULL'),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
# The path as the AGENT resolved it, which is not always the path the
|
||||
# operator typed (environment variables get expanded endpoint-side). Storing
|
||||
# the resolved one is what makes the history honest about what was read.
|
||||
sourcepath = db.Column(db.String(512), nullable=False)
|
||||
|
||||
content = db.Column(db.Text(length=16777215), nullable=True)
|
||||
|
||||
# Size of the file ON DISK, not the size of content. When the tail is
|
||||
# truncated these differ, and that difference is the whole point: it tells
|
||||
# the operator they are looking at the end of something larger.
|
||||
filesizebytes = db.Column(db.BigInteger, nullable=True)
|
||||
contentbytes = db.Column(db.Integer, nullable=True)
|
||||
linecount = db.Column(db.Integer, nullable=True)
|
||||
istruncated = db.Column(db.Boolean, nullable=False,
|
||||
default=False, server_default=sa.false())
|
||||
|
||||
# Last-write time reported by the endpoint. A log that has not changed since
|
||||
# the previous collection is the fastest way to see an app is not logging.
|
||||
filemodifiedat = db.Column(db.DateTime, nullable=True)
|
||||
|
||||
# Username snapshot alongside the id: the id alone becomes unreadable if the
|
||||
# user is later removed, and this history is read long after collection.
|
||||
collectedbyuserid = db.Column(
|
||||
db.Integer,
|
||||
db.ForeignKey('users.userid', ondelete='SET NULL'),
|
||||
nullable=True,
|
||||
)
|
||||
collectedbyusername = db.Column(db.String(100), nullable=True)
|
||||
collectedat = db.Column(db.DateTime, nullable=False,
|
||||
default=datetime.utcnow, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
# Serves both hot paths: listing an asset's history newest-first, and
|
||||
# finding the oldest rows to prune on insert.
|
||||
db.Index('idx_endpointlogcollections_assetcollected',
|
||||
'assetid', 'collectedat'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return (f"<EndpointLogCollection asset={self.assetid} "
|
||||
f"{self.sourcepath}>")
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
"""A download filename derived from the collected path."""
|
||||
raw = (self.sourcepath or '').replace('/', '\\')
|
||||
base = raw.rsplit('\\', 1)[-1].strip() or 'collected.log'
|
||||
# Keep it to characters that survive a Content-Disposition header and a
|
||||
# Windows save dialog without quoting games.
|
||||
safe = re.sub(r'[^A-Za-z0-9._-]', '_', base)[:100]
|
||||
stamp = (self.collectedat or datetime.utcnow()).strftime('%Y%m%d-%H%M%S')
|
||||
return f'{stamp}_{safe}'
|
||||
|
||||
def to_dict(self, includecontent=False):
|
||||
"""Serialise a collection.
|
||||
|
||||
Content is opt-in for the same reason resultjson is on EndpointCommand:
|
||||
the history list renders 20 rows and none of them need the log body,
|
||||
which may be hundreds of kilobytes each.
|
||||
"""
|
||||
data = {
|
||||
'endpointlogcollectionid': self.endpointlogcollectionid,
|
||||
'assetid': self.assetid,
|
||||
'endpointcommandid': self.endpointcommandid,
|
||||
'sourcepath': self.sourcepath,
|
||||
'filesizebytes': self.filesizebytes,
|
||||
'contentbytes': self.contentbytes,
|
||||
'linecount': self.linecount,
|
||||
'istruncated': bool(self.istruncated),
|
||||
'filemodifiedat': _utciso(self.filemodifiedat),
|
||||
'collectedbyuserid': self.collectedbyuserid,
|
||||
'collectedbyusername': self.collectedbyusername,
|
||||
'collectedat': _utciso(self.collectedat),
|
||||
'filename': self.filename,
|
||||
}
|
||||
if includecontent:
|
||||
data['content'] = self.content or ''
|
||||
return data
|
||||
|
||||
@@ -206,7 +206,8 @@ class EndpointsPlugin(BasePlugin):
|
||||
'already registered and allowlisted on the agent)', 'endpoints'),
|
||||
('endpoints.updateapp',
|
||||
'Push application installers to endpoints (runs a pinned '
|
||||
'installer as SYSTEM)', 'endpoints'),
|
||||
'installer as SYSTEM) and collect plaintext log files from '
|
||||
'endpoints by path (read-only, text files only)', 'endpoints'),
|
||||
# Blunt on purpose. This is not "system-level changes" - it is
|
||||
# arbitrary code execution as SYSTEM unless the script descriptor
|
||||
# pins a user context. Whoever grants it must see that here.
|
||||
@@ -503,7 +504,12 @@ class EndpointsPlugin(BasePlugin):
|
||||
},
|
||||
'meta': [
|
||||
{'key': 'detail'},
|
||||
{'key': 'eventtime', 'label': 'At', 'format': 'date'},
|
||||
# datetime, not date: 'date' renders via
|
||||
# toLocaleDateString and drops the clock time entirely,
|
||||
# which made every event on a given day look
|
||||
# simultaneous. Ordering a boot against the logon that
|
||||
# followed it is the whole point of this feed.
|
||||
{'key': 'eventtime', 'label': 'At', 'format': 'datetime'},
|
||||
],
|
||||
# Expands in place on click. The message body is several
|
||||
# hundred words of Windows boilerplate around the one or
|
||||
|
||||
Reference in New Issue
Block a user