Merge the wilm delivery work into main

Ben Baker's installed-software collection, software delivery (run_script and
update_app), agentless adoption tool and command-queue hardening, with his
authorship preserved.

Reviewed before promotion. Fixed: the endpoints plugin tests were never collected
by pytest, the SBOM snapshot table's key would exhaust on MySQL, ingest did not
truncate two columns strict MySQL would reject, and update_app's version branch
pinned a digest for a file that route never serves. Agent packaging moved off
require_role('admin') onto a new endpoints.agentpackage permission.

Contract 0.27.0. Full suite 2231, frontend 314.

tests/test_plugins/test_endpoint_applications_model.py

@@ -0,0 +1,87 @@
+"""The installed-software snapshot table: it must actually accept inserts.
+
+This table is rewritten wholesale on every endpoint check-in - delete the
+asset's rows, insert the current inventory - so its surrogate key climbs at
+(endpoints x installed apps) per report and never reuses a value. On MySQL a
+32-bit key exhausts and inserts stop working permanently.
+
+Widening it to BigInteger broke every insert on SQLite, because SQLite only
+autoincrements a column typed exactly INTEGER PRIMARY KEY. Neither fault was
+caught by the suite, for the same reason: nothing inserted into this table at
+all. This does.
+"""
+from shopdb.api import db
+from plugins.endpoints.models.endpoint import EndpointApplication
+
+
+def test_rows_insert_and_the_key_autoincrements(app):
+    with app.app_context():
+        db.create_all()
+        first = EndpointApplication(assetid=1, displayname='Probe', displayversion='1.0')
+        second = EndpointApplication(assetid=1, displayname='Probe 2', displayversion='2.0')
+        db.session.add_all([first, second])
+        db.session.commit()
+
+        assert first.endpointapplicationid is not None
+        assert second.endpointapplicationid is not None
+        assert second.endpointapplicationid != first.endpointapplicationid
+
+
+def test_the_key_is_wide_enough_for_a_table_that_is_rewritten_every_cycle(app):
+    """BIGINT on MySQL, INTEGER on SQLite - both halves matter.
+
+    Dropping the variant breaks SQLite inserts; dropping the BigInteger breaks
+    MySQL years later, which is worse because it is silent until it is not.
+    """
+    from sqlalchemy.dialects import mysql, sqlite
+
+    with app.app_context():
+        column = EndpointApplication.__table__.c.endpointapplicationid
+        assert 'BIGINT' in str(column.type.compile(dialect=mysql.dialect()))
+        assert 'INTEGER' in str(column.type.compile(dialect=sqlite.dialect()))
+
+
+def test_over_long_registry_values_are_clamped_not_fatal(app):
+    """One malformed Uninstall entry must not cost the whole check-in.
+
+    On MySQL in strict mode an over-long value raises Data too long and fails the
+    transaction, taking telemetry and events down with it - so that endpoint goes
+    silent until somebody uninstalls the offending program. SQLite does not
+    enforce declared lengths, so the suite would never have shown it.
+    """
+    from plugins.endpoints.plugin import EndpointsPlugin
+
+    with app.app_context():
+        db.create_all()
+        warnings = []
+        stored = EndpointsPlugin()._replace_applications(
+            assetid=4242,
+            applications=[{
+                'displayname': 'A' * 400,
+                'displayversion': 'V' * 400,
+                'publisher': 'P' * 400,
+            }],
+            warnings=warnings,
+        )
+        db.session.commit()
+
+        assert stored == 1
+        row = EndpointApplication.query.filter_by(assetid=4242).one()
+        assert len(row.displayname) == 255
+        assert len(row.displayversion) == 128
+        assert len(row.publisher) == 255
+
+
+def test_a_blank_value_becomes_null_rather_than_an_empty_string(app):
+    from plugins.endpoints.plugin import EndpointsPlugin
+
+    with app.app_context():
+        db.create_all()
+        EndpointsPlugin()._replace_applications(
+            assetid=4243,
+            applications=[{'displayname': 'Thing', 'publisher': '   '}],
+            warnings=[],
+        )
+        db.session.commit()
+        row = EndpointApplication.query.filter_by(assetid=4243).one()
+        assert row.publisher is None

tests/test_plugins/test_endpoint_delivery_descriptor.py

@@ -0,0 +1,81 @@
+"""An update_app delivery must pin the bytes it actually delivers.
+
+The agent verifies the downloaded file against the descriptor's sha256 and fails
+closed on a mismatch - which is worthless if the descriptor carries no digest,
+and misleading if it carries a digest for a different file.
+
+Both were true of the version: branch. It pinned AppVersion.sha256, but
+/api/endpoints/apppackage/<appversionid> resolves the version to its application
+and serves application.installpath, so the digest described an artifact that
+route never sends; where the version had no sha256 the descriptor carried none
+at all and the agent had nothing to check.
+
+It also ended the URL at the id, so an MSI arrived with no extension and the
+agent's Path.GetExtension could not route it through msiexec.
+"""
+import pytest
+
+from shopdb.api import db
+from shopdb.core.models import Application, AppVersion
+
+
+@pytest.fixture
+def hosted_app(app, request):
+    # Unique per test: appname is unique-constrained and the database outlives
+    # a single test here, so a fixed name collides on the second use.
+    with app.app_context():
+        db.create_all()
+        application = Application(
+            appname='Widget-%s' % request.node.name[-24:],
+            installpath='/api/applications/package/widget-setup.msi',
+            installersha256='a' * 64,
+        )
+        db.session.add(application)
+        db.session.commit()
+        version = AppVersion(appid=application.appid, version='2.1', sha256=None)
+        db.session.add(version)
+        db.session.commit()
+        yield application.appid, version.appversionid
+
+
+def test_a_version_delivery_carries_a_digest_even_when_the_version_has_none(app, hosted_app):
+    from plugins.endpoints.api.routes import _resolve_update_app
+
+    appid, appversionid = hosted_app
+    with app.app_context():
+        descriptor, error = _resolve_update_app(f'version:{appversionid}')
+
+    assert error is None, error
+    # Not None: this is the whole point. A blank pin makes the agent fail open.
+    assert descriptor['sha256'] == 'a' * 64
+
+
+def test_a_version_delivery_keeps_the_installer_extension(app, hosted_app):
+    """Without the filename an MSI is not routed through msiexec."""
+    from plugins.endpoints.api.routes import _resolve_update_app
+
+    appid, appversionid = hosted_app
+    with app.app_context():
+        descriptor, _ = _resolve_update_app(f'version:{appversionid}')
+
+    assert descriptor['installpath'].endswith('.msi')
+
+
+def test_a_package_with_no_recorded_digest_is_refused_not_delivered_unverified(app):
+    """Refusing to enqueue beats delivering something nothing can verify."""
+    from plugins.endpoints.api.routes import _resolve_update_app
+
+    with app.app_context():
+        db.create_all()
+        application = Application(appname='Unhashed-%s' % id(app),
+                                  installpath='/api/applications/package/x.exe')
+        db.session.add(application)
+        db.session.commit()
+        version = AppVersion(appid=application.appid, version='1.0', sha256=None)
+        db.session.add(version)
+        db.session.commit()
+
+        descriptor, error = _resolve_update_app(f'version:{version.appversionid}')
+
+    assert descriptor is None
+    assert 'SHA-256' in error

tests/test_plugins/test_endpoints_plugin.py

@@ -0,0 +1,251 @@
+"""Endpoints plugin smoke + collector-contract tests.
+
+Asserts the plugin loads cleanly, satisfies the framework contract, and that
+its ADR-006 collector schema is well formed. Extend with DB-backed ingest tests
+as the plugin grows.
+"""
+
+from flask import Blueprint
+
+from plugins.endpoints.plugin import EndpointsPlugin, CURATED_EVENT_IDS
+from plugins.endpoints.eventcatalog import EVENT_CATALOG, CRITICAL_EVENT_IDS
+from plugins.endpoints.commands import (
+    COMMAND_ALLOWLIST, is_allowed_action, elevated_permission_for,
+    requires_target,
+)
+
+
+def test_endpoints_plugin_meta_is_valid():
+    """meta returns a PluginMeta with the expected name and prefix."""
+    plugin = EndpointsPlugin()
+    assert plugin.meta.name == 'endpoints'
+    assert plugin.meta.api_prefix == '/api/endpoints'
+    assert 'computers' in plugin.meta.dependencies
+
+
+def test_endpoints_plugin_get_blueprint_returns_blueprint():
+    """get_blueprint returns a Flask Blueprint, not None."""
+    plugin = EndpointsPlugin()
+    assert isinstance(plugin.get_blueprint(), Blueprint)
+
+
+def test_endpoints_plugin_get_models_returns_models():
+    """get_models returns the two telemetry tables."""
+    plugin = EndpointsPlugin()
+    models = plugin.get_models()
+    assert len(models) >= 2
+    for model in models:
+        assert hasattr(model, '__tablename__')
+
+
+def test_endpoints_collector_schema_identity_is_hostname():
+    """ADR-006 schema declares hostname as the identity field."""
+    plugin = EndpointsPlugin()
+    schema = plugin.get_collector_schema()
+    assert schema is not None
+    assert schema['identityfield'] == 'hostname'
+    assert 'hostname' in schema['fields']['required']
+
+
+def test_endpoints_curated_event_ids_include_core_security_ids():
+    """The curated catalog covers the agreed logon/stability IDs."""
+    for eventid in (4624, 4625, 4634, 4672, 4697, 4698, 41, 4618, 55, 7011,
+                    10016, 1001):
+        assert eventid in CURATED_EVENT_IDS
+
+
+def test_curated_ids_match_the_event_catalog():
+    """Ingest filter and display catalog cannot drift apart."""
+    assert CURATED_EVENT_IDS == frozenset(EVENT_CATALOG)
+
+
+def test_critical_event_ids_are_a_subset_of_the_catalog():
+    """Dashboard stability card only queries known critical IDs."""
+    assert set(CRITICAL_EVENT_IDS)
+    assert set(CRITICAL_EVENT_IDS).issubset(set(EVENT_CATALOG))
+
+
+def test_endpoints_asset_panels_are_computer_scoped():
+    """Every asset panel targets computer assets only, with an endpoint."""
+    plugin = EndpointsPlugin()
+    panels = plugin.get_asset_panels()
+    assert len(panels) == 6
+    for panel in panels:
+        assert panel['assettypes'] == ['computer']
+        assert panel['endpoint'].startswith('/api/endpoints/asset/{assetid}')
+        assert panel['render'] in ('keyvalue', 'list', 'table')
+
+
+def test_services_and_processes_are_split_into_two_filtered_cards():
+    """The single services card is now two cards filtered by row kind, both
+    reading the same services snapshot so a future top-20 collector is just
+    another filtered panel over the same feed."""
+    plugin = EndpointsPlugin()
+    panels = {p['id']: p for p in plugin.get_asset_panels()}
+    services = panels['endpoints-tracked-services']
+    processes = panels['endpoints-tracked-processes']
+    assert services['title'] == 'Tracked Services'
+    assert processes['title'] == 'Tracked Processes'
+    assert services['filter'] == {'key': 'kind', 'equals': 'service'}
+    assert processes['filter'] == {'key': 'kind', 'equals': 'process'}
+    # Both slice one endpoint feed.
+    assert services['endpoint'] == processes['endpoint']
+    assert services['endpoint'].endswith('/services')
+    # Command wiring carries over to both cards.
+    assert services['permission'] == 'endpoints.command'
+    assert processes['permission'] == 'endpoints.command'
+
+
+def test_activity_card_starts_collapsed_and_sits_below_live_state():
+    """The activity feed is history, so it must not push live state down.
+
+    Renamed from the events card when events and commands merged into one feed.
+    It is no longer the LAST panel - installed software sits below it - so the
+    assertion is that it ranks below every live-state panel rather than being
+    the maximum, which would break again the next time a reference panel is
+    added underneath.
+    """
+    plugin = EndpointsPlugin()
+    panels = plugin.get_asset_panels()
+    activity = next(p for p in panels if p['id'] == 'endpoints-activity')
+    # NOT collapsed by default, unlike the events card it replaced. The feed
+    # surfaces critical events, and a card that hides the one thing worth seeing
+    # is the reason the health dot and the visible rows disagreed. Collapsible,
+    # so a reader can put it away.
+    assert activity.get('collapsible') is True
+    assert activity.get('collapsedDefault') is not True
+    live = [p['position'] for p in panels
+            if p['id'] in ('endpoints-status', 'endpoints-tracked-services',
+                           'endpoints-tracked-processes', 'endpoints-interfaces')]
+    assert activity['position'] > max(live)
+
+
+def test_endpoints_dashboard_widgets_declared():
+    """Quiet + stability exception cards are declared and hide when empty."""
+    plugin = EndpointsPlugin()
+    widgets = plugin.get_dashboard_widgets()
+    ids = {w['id'] for w in widgets}
+    assert {'endpoints-quiet', 'endpoints-stability'} <= ids
+    for widget in widgets:
+        assert widget['render'] == 'exceptions'
+        assert widget['empty'] == 'hide'
+        # The fleet-wide cards read the whole roster, so they are gated on
+        # endpoints.fleet, not the per-endpoint endpoints.view a guest may hold.
+        assert widget['permission'] == 'endpoints.fleet'
+
+
+def test_endpoints_navigation_item_points_at_fleet_view():
+    """The fleet list view is exposed as a single /endpoints nav item."""
+    plugin = EndpointsPlugin()
+    items = plugin.get_navigation_items()
+    routes = {item['route'] for item in items}
+    assert '/endpoints' in routes
+
+
+def test_system_restart_is_allowlisted_and_policy_selectable():
+    """Restart is a real allowlisted action, so the policy builder (which
+    reads COMMAND_ALLOWLIST) surfaces it as a selectable action."""
+    assert is_allowed_action('system_restart')
+    assert 'system_restart' in COMMAND_ALLOWLIST
+    assert COMMAND_ALLOWLIST['system_restart']['label'] == 'Restart computer'
+
+
+def test_system_restart_requires_the_elevated_restart_permission():
+    """Restart maps to endpoints.restart so the enqueue route can demand a
+    heavier grant than the routine endpoints.command toggle. Routine actions
+    carry no elevated requirement."""
+    assert elevated_permission_for('system_restart') == 'endpoints.restart'
+    assert elevated_permission_for('mtconnect_start') is None
+
+
+def test_endpoints_declares_the_restart_permission():
+    """The dedicated restart permission is declared for role-grid seeding."""
+    plugin = EndpointsPlugin()
+    permissions = {name for name, *_ in plugin.get_permissions()}
+    assert 'endpoints.restart' in permissions
+    assert 'endpoints.command' in permissions
+
+
+def test_collect_now_is_allowlisted_targetless_and_routine():
+    """collect_now forces an off-cadence telemetry report. It is allowlisted so
+    the enqueue route accepts it, carries no wire target (fixed action), and
+    needs no elevated grant beyond endpoints.command."""
+    assert is_allowed_action('collect_now')
+    assert 'collect_now' in COMMAND_ALLOWLIST
+    assert not requires_target('collect_now')
+    assert elevated_permission_for('collect_now') is None
+
+
+def test_run_script_and_update_app_are_allowlisted_delivery_actions():
+    """The two software-delivery actions are allowlisted with a targetkind so
+    the enqueue route accepts them and carries their subject on the wire:
+    run_script carries a script name, update_app carries an appversionid."""
+    for action in ('run_script', 'update_app'):
+        assert is_allowed_action(action)
+        assert action in COMMAND_ALLOWLIST
+        assert requires_target(action)
+    assert COMMAND_ALLOWLIST['run_script']['targetkind'] == 'script'
+    assert COMMAND_ALLOWLIST['update_app']['targetkind'] == 'app'
+
+
+def test_delivery_actions_require_the_elevated_deploy_permission():
+    """run_script and update_app both demand endpoints.deploy ON TOP OF
+    endpoints.command, so pushing software is a strictly heavier grant than a
+    routine service/process toggle."""
+    assert elevated_permission_for('run_script') == 'endpoints.deploy'
+    assert elevated_permission_for('update_app') == 'endpoints.deploy'
+
+
+def test_endpoints_declares_the_deploy_permission():
+    """The dedicated software-delivery permission is declared for role seeding."""
+    plugin = EndpointsPlugin()
+    permissions = {name for name, *_ in plugin.get_permissions()}
+    assert 'endpoints.deploy' in permissions
+
+
+def test_known_payload_keys_cover_persisted_and_relayed_fields():
+    """The ingest allow-set must list every key the handler consumes today, so
+    those keys are never mislogged as 'unpersisted'. Anything the agent sends
+    outside this set is logged for observability before columns exist."""
+    from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS
+
+    persisted_or_relayed = {
+        'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype',
+        'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress',
+        'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
+        'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces',
+        'scheduledtasks', 'events',
+    }
+    assert persisted_or_relayed <= KNOWN_PAYLOAD_KEYS
+
+
+def test_new_hardware_telemetry_keys_are_logged_not_silently_known():
+    """Disk/bios/memory inventory is collected at the agent before the server
+    persists it. Those keys must be OUTSIDE the known set so ingest logs them
+    rather than dropping them silently."""
+    from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS
+
+    for key in ('disks', 'bios', 'memory', 'processor', 'osinfo', 'domain'):
+        assert key not in KNOWN_PAYLOAD_KEYS
+
+    # 'applications' USED to be in this list. It is persisted now - that is what
+    # the installed-software work added - so it belongs in the known set, and
+    # asserting its absence would fail the moment the feature it tests works.
+    assert 'applications' in KNOWN_PAYLOAD_KEYS
+
+
+
+def test_telemetrymeta_route_is_registered_for_freshness_gating():
+    """The WebUI gates panel refreshes on lastreportedat advancing, so the
+    lightweight freshness probe must be exposed on the plugin blueprint."""
+    from flask import Flask
+
+    application = Flask(__name__)
+    # A url_prefix is REQUIRED here: the fleet list registers as route(''), which
+    # Flask rejects as a bare rule and accepts under a prefix. That is deliberate
+    # - CONTRIBUTING says API routes carry no trailing slash - so the test has to
+    # mount the blueprint the way the app does.
+    application.register_blueprint(EndpointsPlugin().get_blueprint(),
+                                   url_prefix='/api/endpoints')
+    rules = {rule.rule for rule in application.url_map.iter_rules()}
+    assert any(rule.endswith('/asset/<int:asset_id>/telemetrymeta') for rule in rules)

tests/test_plugins/test_printedparts_plugin.py

@@ -0,0 +1,30 @@
+"""Printedparts plugin smoke tests.
+
+Asserts the plugin loads cleanly and satisfies the framework contract.
+Replace and extend with domain tests as you build the plugin out.
+"""
+
+from plugins.printedparts.plugin import PrintedpartsPlugin
+
+
+def test_printedparts_plugin_meta_is_valid():
+    """PrintedpartsPlugin.meta returns a PluginMeta with the expected name."""
+    plugin = PrintedpartsPlugin()
+    assert plugin.meta.name == 'printedparts'
+    assert plugin.meta.api_prefix == '/api/printedparts'
+
+
+def test_printedparts_plugin_get_blueprint_returns_blueprint():
+    """get_blueprint returns a Flask Blueprint, not None."""
+    from flask import Blueprint
+    plugin = PrintedpartsPlugin()
+    assert isinstance(plugin.get_blueprint(), Blueprint)
+
+
+def test_printedparts_plugin_get_models_returns_a_model():
+    """get_models returns a list with at least one SQLAlchemy model."""
+    plugin = PrintedpartsPlugin()
+    models = plugin.get_models()
+    assert len(models) >= 1
+    for model in models:
+        assert hasattr(model, '__tablename__')