ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add endpoint monitor catalog and warn-only drift reconciler

Browse Source 
Operators need to see when a machine's running services or processes fall outside the desired state defined by policy. This adds a reusable monitor catalog (shipped read-only rows plus user-built local rows), computes drift server-side on agent check-in, and surfaces a warning on the endpoint panels. Remediation stays manual for now; automation can follow once demand is proven. Includes migrations 0006-0008 and the monitors API client.
This commit is contained in:
benjamin.baker
2026-09-01 15:44:03 -04:00
parent 1a1bf0640f
commit 42db980ca9
23 changed files with 2054 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
tests/test_plugins/test_endpoints_policy_monitors.py Normal file
View File

@@ -0,0 +1,199 @@
"""Policy-attached monitors surface in the Tracked Services/Processes panel.
A monitor attached to a PC type through an active Business Policy (over a scoped
Configuration Policy) must appear in GET /api/endpoints/asset/<id>/services even
before the agent has reported that process/service, so the PC Detail Monitors
tab shows the intended coverage as an Unknown row.
"""
from shopdb.core.models import Asset, AssetType, Setting
SERVICES_URL = '/api/endpoints/asset/{}/services'
def _computer(db, hostname, computertype):
"""Create a computer Asset + Computer of the given PC type; return the asset."""
from plugins.computers.models import Computer, ComputerType
atype = AssetType.query.filter_by(assettype='computer').first()
if not atype:
atype = AssetType(assettype='computer')
db.session.add(atype)
db.session.flush()
ctype = ComputerType.query.filter_by(computertype=computertype).first()
if not ctype:
ctype = ComputerType(computertype=computertype)
db.session.add(ctype)
db.session.flush()
asset = Asset(assetnumber=hostname, assettypeid=atype.assettypeid,
isactive=True)
db.session.add(asset)
db.session.flush()
comp = Computer(assetid=asset.assetid, hostname=hostname,
computertypeid=ctype.computertypeid)
db.session.add(comp)
db.session.flush()
return asset, ctype
def _attach_monitor_to_pctype(db, monitor, computertypeid):
"""Create a Configuration Policy scoped to the PC type carrying the monitor,
wrapped in an active Business Policy, with the feature flag enabled."""
from plugins.computers.models import (
ConfigurationPolicy, ConfigurationPolicyAction,
BusinessPolicy, BusinessPolicyItem,
)
db.session.add(Setting(key='computers_configpolicies_enabled', value='true'))
policy = ConfigurationPolicy(
name='Edge coverage',
scopejson={'type': 'pctype', 'id': computertypeid},
)
db.session.add(policy)
db.session.flush()
db.session.add(ConfigurationPolicyAction(
configurationpolicyid=policy.configurationpolicyid,
action=f'monitor:{monitor.endpointmonitorid}',
mode='gate',
))
business = BusinessPolicy(name='Test Policy', priority=100)
db.session.add(business)
db.session.flush()
db.session.add(BusinessPolicyItem(
businesspolicyid=business.businesspolicyid,
configurationpolicyid=policy.configurationpolicyid,
sortorder=0,
))
db.session.commit()
def test_policy_attached_process_monitor_surfaces_as_unknown_row(
client, db, auth_headers):
"""A msedge.exe process monitor attached to a PC type by policy appears in
the services panel as an Unknown process row, even with no agent report."""
from plugins.endpoints.models import EndpointMonitor
asset, ctype = _computer(db, 'F18BCDB4', 'CMM PC')
monitor = EndpointMonitor(
kind='process', displayname='msedge.exe', processname='msedge.exe',
source='local', isactive=True,
)
db.session.add(monitor)
db.session.flush()
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
response = client.get(SERVICES_URL.format(asset.assetid), headers=auth_headers)
assert response.status_code == 200
rows = response.get_json()['data']
edge = next((r for r in rows if r['name'] == 'msedge.exe'), None)
assert edge is not None, 'policy-attached monitor missing from panel'
assert edge['kind'] == 'process'
assert edge['statelabel'] == 'Unknown'
assert edge['statecolor'] == 'gray'
# Admin holds endpoints.command, so the row offers a Start control whose
# wire target is the OS process name.
assert edge['action']['target'] == 'msedge.exe'
assert edge['action']['targetkind'] == 'process'
assert edge['action']['running'] is False
def test_flag_off_hides_policy_monitor_from_panel(client, db, auth_headers):
"""With the feature flag off the panel returns only reported rows (none)."""
from plugins.endpoints.models import EndpointMonitor
asset, ctype = _computer(db, 'F18BCDB5', 'CMM PC')
monitor = EndpointMonitor(
kind='process', displayname='msedge.exe', processname='msedge.exe',
source='local', isactive=True,
)
db.session.add(monitor)
db.session.flush()
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
# Flip the flag off after the helper turned it on.
flag = Setting.query.filter_by(
key='computers_configpolicies_enabled').first()
flag.value = 'false'
db.session.commit()
response = client.get(SERVICES_URL.format(asset.assetid), headers=auth_headers)
assert response.status_code == 200
rows = response.get_json()['data']
assert not any(r['name'] == 'msedge.exe' for r in rows)
RESOLVED_URL = '/api/endpoints/monitors/resolved'
COLLECTOR_KEY = 'testcollectorkey'
def _collector_headers(app):
"""Authorize the agent-facing route on the shared collector key."""
app.config['COLLECTOR_API_KEY'] = COLLECTOR_KEY
return {'X-API-Key': COLLECTOR_KEY}
def test_resolved_monitors_returns_policy_process_for_agent(client, db, app):
"""The agent watch-list route returns a policy-attached process monitor,
shaped as processes[]/services[] of {name, app}, keyed by hostname."""
from plugins.endpoints.models import EndpointMonitor
asset, ctype = _computer(db, 'AGENTPC1', 'CMM PC')
monitor = EndpointMonitor(
kind='process', displayname='Edge', processname='msedge.exe',
source='local', isactive=True,
)
db.session.add(monitor)
db.session.flush()
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
response = client.get(RESOLVED_URL, query_string={'hostname': 'AGENTPC1'},
headers=_collector_headers(app))
assert response.status_code == 200
data = response.get_json()['data']
assert {'name': 'msedge.exe', 'app': 'Edge'} in data['processes']
assert data['services'] == []
def test_resolved_monitors_carries_launchpath_for_process(client, db, app):
"""A process monitor with a launch path set surfaces launchpath on the wire
so the agent can honor a site-provisioned Start path (after validation)."""
from plugins.endpoints.models import EndpointMonitor
asset, ctype = _computer(db, 'AGENTPC3', 'CMM PC')
monitor = EndpointMonitor(
kind='process', displayname='Edge', processname='msedge.exe',
launchpath=r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe',
source='local', isactive=True,
)
db.session.add(monitor)
db.session.flush()
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
response = client.get(RESOLVED_URL, query_string={'hostname': 'AGENTPC3'},
headers=_collector_headers(app))
assert response.status_code == 200
data = response.get_json()['data']
edge = next((p for p in data['processes'] if p['name'] == 'msedge.exe'), None)
assert edge is not None
assert edge['launchpath'] == \
r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'
def test_resolved_monitors_requires_collector_auth(client, db, app):
"""Without the collector key the agent route refuses (no JWT fallback)."""
_computer(db, 'AGENTPC2', 'CMM PC')
app.config['COLLECTOR_API_KEY'] = COLLECTOR_KEY
response = client.get(RESOLVED_URL, query_string={'hostname': 'AGENTPC2'})
assert response.status_code in (401, 403)
def test_resolved_monitors_unknown_hostname_is_404(client, db, app):
"""A hostname with no computer row is a 404, not an empty 200."""
response = client.get(RESOLVED_URL, query_string={'hostname': 'NOSUCHBOX'},
headers=_collector_headers(app))
assert response.status_code == 404

238
tests/test_plugins/test_endpoints_reconciler.py Normal file
View File

@@ -0,0 +1,238 @@
"""Warn-only reconciler: drift computation and its exposure on the panels.
The reconciler runs server-side on agent check-in. For every monitor a site
policy attaches to an endpoint's PC type that is EXPECTED to be running, it flags
DRIFT when the latest snapshot does not report it running. It never starts or
stops anything - drift is surfaced for an operator to act on manually.
These tests cover the pure computer (compute_drift), the check-in write path
(apply_collector_payload persisting driftjson), the panel exposure, and the
observe-only seam (monitor_expects_running) that keeps a non-keepalive monitor
out of the drift list.
"""
from shopdb.core.models import Asset, AssetType, Setting
SERVICES_URL = '/api/endpoints/asset/{}/services'
STATUS_URL = '/api/endpoints/asset/{}/status'
def _computer(db, hostname, computertype):
"""Create a computer Asset + Computer of the given PC type; return the pair."""
from plugins.computers.models import Computer, ComputerType
atype = AssetType.query.filter_by(assettype='computer').first()
if not atype:
atype = AssetType(assettype='computer')
db.session.add(atype)
db.session.flush()
ctype = ComputerType.query.filter_by(computertype=computertype).first()
if not ctype:
ctype = ComputerType(computertype=computertype)
db.session.add(ctype)
db.session.flush()
asset = Asset(assetnumber=hostname, assettypeid=atype.assettypeid,
isactive=True)
db.session.add(asset)
db.session.flush()
comp = Computer(assetid=asset.assetid, hostname=hostname,
computertypeid=ctype.computertypeid)
db.session.add(comp)
db.session.flush()
return asset, ctype
def _attach_monitor_to_pctype(db, monitor, computertypeid):
"""Attach a monitor to a PC type via an active Business/Configuration Policy."""
from plugins.computers.models import (
ConfigurationPolicy, ConfigurationPolicyAction,
BusinessPolicy, BusinessPolicyItem,
)
if not Setting.query.filter_by(
key='computers_configpolicies_enabled').first():
db.session.add(
Setting(key='computers_configpolicies_enabled', value='true'))
policy = ConfigurationPolicy(
name='Coverage {}'.format(monitor.endpointmonitorid),
scopejson={'type': 'pctype', 'id': computertypeid},
)
db.session.add(policy)
db.session.flush()
db.session.add(ConfigurationPolicyAction(
configurationpolicyid=policy.configurationpolicyid,
action='monitor:{}'.format(monitor.endpointmonitorid),
mode='gate',
))
business = BusinessPolicy(name='BP {}'.format(monitor.endpointmonitorid),
priority=100)
db.session.add(business)
db.session.flush()
db.session.add(BusinessPolicyItem(
businesspolicyid=business.businesspolicyid,
configurationpolicyid=policy.configurationpolicyid,
sortorder=0,
))
db.session.commit()
def _process_monitor(db, displayname, processname):
from plugins.endpoints.models import EndpointMonitor
monitor = EndpointMonitor(
kind='process', displayname=displayname, processname=processname,
source='local', isactive=True,
)
db.session.add(monitor)
db.session.flush()
return monitor
# ---------------------------------------------------------------------------
# Pure computer
# ---------------------------------------------------------------------------
def test_compute_drift_flags_attached_monitor_not_reported_running(
client, db, app):
"""An expected-running monitor absent from the snapshot is drift."""
from plugins.endpoints.services.reconciler import compute_drift
asset, ctype = _computer(db, 'DRIFT01', 'CMM PC')
monitor = _process_monitor(db, 'eDNC', 'dncMain.exe')
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
drift = compute_drift(asset.assetid, [])
assert len(drift) == 1
assert drift[0]['name'] == 'dncMain.exe'
assert drift[0]['kind'] == 'process'
assert drift[0]['expected'] == 'running'
assert drift[0]['observed'] == 'notrunning'
def test_compute_drift_empty_when_monitor_reported_running(client, db, app):
"""A monitor the snapshot reports running is not drift."""
from plugins.endpoints.services.reconciler import compute_drift
asset, ctype = _computer(db, 'DRIFT02', 'CMM PC')
monitor = _process_monitor(db, 'eDNC', 'dncMain.exe')
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
reported = [{'name': 'dncMain.exe', 'kind': 'process', 'state': 'running'}]
assert compute_drift(asset.assetid, reported) == []
def test_compute_drift_empty_when_no_monitor_attached(client, db, app):
"""No attached monitors means nothing to reconcile."""
from plugins.endpoints.services.reconciler import compute_drift
asset, _ = _computer(db, 'DRIFT03', 'CMM PC')
assert compute_drift(asset.assetid, []) == []
def test_monitor_expects_running_is_the_observe_only_seam():
"""The 'should be up' decision lives in one function; inactive is excluded."""
from plugins.endpoints.services.reconciler import monitor_expects_running
class _M:
isactive = True
active = _M()
inactive = _M()
inactive.isactive = False
assert monitor_expects_running(active) is True
assert monitor_expects_running(inactive) is False
# ---------------------------------------------------------------------------
# Check-in write path
# ---------------------------------------------------------------------------
def test_checkin_persists_drift_on_endpoint_state(client, db, app):
"""apply_collector_payload caches drift on the state in the same commit."""
from plugins.endpoints.plugin import EndpointsPlugin
from plugins.endpoints.models import EndpointState
asset, ctype = _computer(db, 'CHKIN01', 'CMM PC')
monitor = _process_monitor(db, 'eDNC', 'dncMain.exe')
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
plugin = EndpointsPlugin()
plugin.apply_collector_payload({'hostname': 'CHKIN01', 'services': []})
db.session.commit()
state = EndpointState.query.filter_by(assetid=asset.assetid).first()
assert state.driftjson is not None
import json
drift = json.loads(state.driftjson)
assert [d['name'] for d in drift] == ['dncMain.exe']
def test_checkin_clears_drift_when_process_returns(client, db, app):
"""A later report that shows the process running clears the cached drift."""
from plugins.endpoints.plugin import EndpointsPlugin
from plugins.endpoints.models import EndpointState
import json
asset, ctype = _computer(db, 'CHKIN02', 'CMM PC')
monitor = _process_monitor(db, 'eDNC', 'dncMain.exe')
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
plugin = EndpointsPlugin()
plugin.apply_collector_payload({'hostname': 'CHKIN02', 'services': []})
db.session.commit()
plugin.apply_collector_payload({
'hostname': 'CHKIN02',
'services': [{'name': 'dncMain.exe', 'kind': 'process',
'state': 'running'}],
})
db.session.commit()
state = EndpointState.query.filter_by(assetid=asset.assetid).first()
assert json.loads(state.driftjson) == []
# ---------------------------------------------------------------------------
# Panel exposure
# ---------------------------------------------------------------------------
def test_status_panel_surfaces_drift_warning(client, db, app, auth_headers):
"""The status panel adds a Needs attention field when drift is cached."""
from plugins.endpoints.plugin import EndpointsPlugin
asset, ctype = _computer(db, 'PANEL01', 'CMM PC')
monitor = _process_monitor(db, 'eDNC', 'dncMain.exe')
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
plugin = EndpointsPlugin()
plugin.apply_collector_payload({'hostname': 'PANEL01', 'services': []})
db.session.commit()
response = client.get(STATUS_URL.format(asset.assetid), headers=auth_headers)
assert response.status_code == 200
labels = [f['label'] for f in response.get_json()['data']['fields']]
assert 'Needs attention' in labels
def test_services_panel_marks_drifted_row(client, db, app, auth_headers):
"""The synthetic Unknown row for an expected-running monitor is drift-flagged."""
from plugins.endpoints.plugin import EndpointsPlugin
asset, ctype = _computer(db, 'PANEL02', 'CMM PC')
monitor = _process_monitor(db, 'eDNC', 'dncMain.exe')
_attach_monitor_to_pctype(db, monitor, ctype.computertypeid)
plugin = EndpointsPlugin()
plugin.apply_collector_payload({'hostname': 'PANEL02', 'services': []})
db.session.commit()
response = client.get(SERVICES_URL.format(asset.assetid), headers=auth_headers)
assert response.status_code == 200
rows = response.get_json()['data']
edenc = next((r for r in rows if r['name'] == 'eDNC'), None)
assert edenc is not None
assert edenc.get('drift') is True

141
tests/test_plugins/test_endpoints_shipped_monitor_policy.py Normal file
View File

@@ -0,0 +1,141 @@
"""The shipped shopfloor app monitors ride a read-only Site Policy.
Migration endpoints0007shippedpolicy offloads the eDNC/UDC/MTConnect monitors
from the agent's hardcoded appsettings.json into a product-owned policy bundle:
a shipped ConfigurationPolicy "Shopfloor App Monitors" (global scope, one
desired-state monitor action per shipped monitor) wrapped in a shipped
BusinessPolicy "RPCA Shopfloor App Monitors".
Run the endpoints chain against a fresh engine, alongside the computers policy
tables the bundle writes into, and assert the bundle exists, is source='shipped'
(read-only), and attaches every shipped monitor as a desired/running:true action.
"""
import importlib.util
import json
from pathlib import Path
from sqlalchemy import create_engine, inspect, text
from alembic.migration import MigrationContext
from alembic.operations import Operations
_ROOT = Path(__file__).resolve().parent.parent.parent
_ENDPOINTS_VERSIONS = _ROOT / 'plugins' / 'endpoints' / 'migrations' / 'versions'
_COMPUTERS_VERSIONS = _ROOT / 'plugins' / 'computers' / 'migrations' / 'versions'
CONFIG_POLICY_NAME = 'Shopfloor App Monitors'
BUSINESS_POLICY_NAME = 'RPCA Shopfloor App Monitors'
def _load(path, prefix):
spec = importlib.util.spec_from_file_location(prefix + path.stem, str(path))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _run_bundle_migrations(connection):
"""Build the policy tables (computers 0004), the monitor catalog (endpoints
0006) and the shipped bundle (endpoints 0007) on one engine, in order.
The core baseline creates every plugin's tables before per-plugin chains run,
so on a real deploy the computers policy tables already exist when endpoints
0007 runs. Here we stand up exactly the three migrations the bundle depends
on - the earlier endpoints revisions carry FKs to core tables that a bare
engine does not have, and are irrelevant to what 0007 seeds.
"""
operations = Operations(MigrationContext.configure(connection))
operations._install_proxy()
try:
_load(_COMPUTERS_VERSIONS / '0004_policies.py',
'computers_mig_').upgrade()
_load(_ENDPOINTS_VERSIONS / '0006_endpoint_monitors.py',
'endpoints_mig_0006_').upgrade()
_load(_ENDPOINTS_VERSIONS / '0007_shipped_monitor_policy.py',
'endpoints_mig_0007_').upgrade()
finally:
operations._remove_proxy()
connection.commit()
def test_shipped_bundle_seeds_read_only_site_policy():
engine = create_engine('sqlite://')
with engine.connect() as connection:
_run_bundle_migrations(connection)
config = connection.execute(text(
'SELECT configurationpolicyid, source, scopejson '
'FROM configurationpolicies WHERE name = :name'),
{'name': CONFIG_POLICY_NAME}).first()
assert config is not None, 'shipped Configuration Policy was not seeded'
config_id, config_source, scopejson = config
assert config_source == 'shipped'
assert json.loads(scopejson) == {'type': 'true'}
business = connection.execute(text(
'SELECT businesspolicyid, source FROM businesspolicies '
'WHERE name = :name'), {'name': BUSINESS_POLICY_NAME}).first()
assert business is not None, 'shipped Business Policy was not seeded'
business_id, business_source = business
assert business_source == 'shipped'
item = connection.execute(text(
'SELECT configurationpolicyid FROM businesspolicyitems '
'WHERE businesspolicyid = :id'), {'id': business_id}).first()
assert item is not None and item[0] == config_id
def test_shipped_bundle_attaches_every_shipped_monitor():
engine = create_engine('sqlite://')
with engine.connect() as connection:
_run_bundle_migrations(connection)
monitor_ids = {row[0] for row in connection.execute(text(
'SELECT endpointmonitorid FROM endpointmonitors '
"WHERE source = 'shipped'"))}
assert monitor_ids, 'no shipped monitors to attach'
config_id = connection.execute(text(
'SELECT configurationpolicyid FROM configurationpolicies '
'WHERE name = :name'), {'name': CONFIG_POLICY_NAME}).scalar()
actions = connection.execute(text(
'SELECT action, mode, desiredstate FROM configurationpolicyactions '
'WHERE configurationpolicyid = :id'), {'id': config_id}).all()
attached_ids = set()
for action, mode, desiredstate in actions:
assert action.startswith('monitor:')
# Gate, not desired: the shipped bundle reports state and exposes
# Start/Stop in the UI, but never auto-starts a process. A desired
# running state is a deliberate per-monitor opt-in, not shipped.
assert mode == 'gate'
assert desiredstate is None
attached_ids.add(int(action[len('monitor:'):]))
assert attached_ids == monitor_ids
def test_shipped_bundle_is_idempotent():
engine = create_engine('sqlite://')
with engine.connect() as connection:
_run_bundle_migrations(connection)
# Re-run only the shipped-bundle migration; it must not duplicate rows.
operations = Operations(MigrationContext.configure(connection))
operations._install_proxy()
try:
_load(_ENDPOINTS_VERSIONS / '0007_shipped_monitor_policy.py',
'endpoints_mig_rerun_').upgrade()
finally:
operations._remove_proxy()
connection.commit()
config_count = connection.execute(text(
'SELECT COUNT(*) FROM configurationpolicies WHERE name = :name'),
{'name': CONFIG_POLICY_NAME}).scalar()
business_count = connection.execute(text(
'SELECT COUNT(*) FROM businesspolicies WHERE name = :name'),
{'name': BUSINESS_POLICY_NAME}).scalar()
assert config_count == 1
assert business_count == 1