Merge the wilm delivery work into main

Ben Baker's installed-software collection, software delivery (run_script and
update_app), agentless adoption tool and command-queue hardening, with his
authorship preserved.

Reviewed before promotion. Fixed: the endpoints plugin tests were never collected
by pytest, the SBOM snapshot table's key would exhaust on MySQL, ingest did not
truncate two columns strict MySQL would reject, and update_app's version branch
pinned a digest for a file that route never serves. Agent packaging moved off
require_role('admin') onto a new endpoints.agentpackage permission.

Contract 0.27.0. Full suite 2231, frontend 314.

shopdb/core/api/__init__.py

@@ -14,6 +14,7 @@ from .operatingsystems import operatingsystems_bp
 from .dashboard import dashboard_bp
 from .dashboarddefaults import dashboarddefaults_bp
 from .applications import applications_bp
+from .scripts import scripts_bp
 from .supportteams import supportteams_bp
 from .search import search_bp
 from .reports import reports_bp
@@ -42,6 +43,7 @@ __all__ = [
     'dashboard_bp',
     'dashboarddefaults_bp',
     'applications_bp',
+    'scripts_bp',
     'supportteams_bp',
     'search_bp',
     'reports_bp',

shopdb/core/api/applications.py

@@ -1,6 +1,7 @@
 """Applications API endpoints."""
 
 import glob
+import hashlib
 import os
 
 from flask import Blueprint, request, current_app
@@ -47,6 +48,20 @@ def _apppackage_dir():
     return os.path.join(current_app.instance_path, 'applicationpackages')
 
 
+def _sha256_of(stream):
+    """Hex SHA-256 of an upload stream, read in chunks, cursor left rewound.
+
+    The agent pins a version to exact bytes; computing the digest here means
+    the server is the one source of truth for what those bytes are.
+    """
+    digest = hashlib.sha256()
+    stream.seek(0)
+    for chunk in iter(lambda: stream.read(1048576), b''):
+        digest.update(chunk)
+    stream.seek(0)
+    return digest.hexdigest()
+
+
 def _replace_upload(directory, stem, ext, upload):
     """Save one file per application, replacing any prior extension.
 
@@ -322,7 +337,9 @@ def create_version(app_id: int):
         appid=app_id,
         version=data['version'],
         releasedate=data.get('releasedate'),
-        notes=data.get('notes')
+        notes=data.get('notes'),
+        sha256=data.get('sha256'),
+        silentargs=data.get('silentargs')
     )
 
     db.session.add(version)
@@ -604,16 +621,37 @@ def upload_application_package(app_id: int):
 
     filename = _replace_upload(_apppackage_dir(), f'application-{app_id}', ext, upload)
     app.installpath = '{}{}'.format(APP_PACKAGE_URL_PREFIX, filename)
+
+    # The endpoint installer push pins bytes by digest; compute it once here so
+    # the server, not the agent, decides what the correct payload is. Stamp it
+    # on the app itself so an update_app push can resolve an install source by
+    # app alone (no AppVersion row). If the caller also names a version, stamp
+    # it there too; otherwise return it so the version-edit flow can persist it.
+    sha256 = _sha256_of(upload.stream)
+    app.installersha256 = sha256
+    stamped_version = None
+    appversionid = request.form.get('appversionid', type=int)
+    if appversionid:
+        stamped_version = db.session.get(AppVersion, appversionid)
+        if not stamped_version or stamped_version.appid != app_id:
+            return error_response(
+                ErrorCodes.VALIDATION_ERROR,
+                'appversionid {} does not belong to this application'.format(appversionid))
+        stamped_version.sha256 = sha256
+
     db.session.commit()
 
     AuditLog.log('updated', 'Application', entityid=app_id,
                  entityname=app.appname,
                  changes={'installpath': {'new': app.installpath},
                           'uploadedfilename': {'new': upload.filename},
+                          'sha256': {'new': sha256},
                           'bytes': {'new': size}})
     db.session.commit()
     return success_response(
-        {**app.to_dict(), 'uploadedfilename': upload.filename, 'bytes': size},
+        {**app.to_dict(), 'uploadedfilename': upload.filename, 'bytes': size,
+         'sha256': sha256,
+         'appversionid': stamped_version.appversionid if stamped_version else None},
         message='Installer uploaded')
 
 

shopdb/core/api/dashboard.py

@@ -38,6 +38,19 @@ def _count_by_type(assettype):
     return query.count()
 
 
+def _inactive_excluded():
+    """Counted-type assets held back by isactive, i.e. soft-deleted.
+
+    Reported so the page can account for itself. A total that silently drops
+    rows is the reason somebody compares it with a list page and finds a
+    difference they cannot explain.
+    """
+    return (db.session.query(Asset).join(AssetType)
+            .filter(Asset.isactive == False,
+                    AssetType.assettype.in_(COUNTEDTYPES))
+            .count())
+
+
 COUNTEDTYPES = ('machine', 'computer', 'network_device', 'printer',
                 'measuring_tool')
 
@@ -69,6 +82,10 @@ def _countedassets():
 @require_permission('assets.view')
 def get_dashboard():
     """Get dashboard summary data (asset-based)."""
+    # Imported here, as in the helpers above: the dualpath service reaches back
+    # into models, so a module-level import is circular.
+    from shopdb.core.services.dualpath import dualpath_single_machine_enabled
+
     machine_count = _count_by_type('machine')
     pc_count = _count_by_type('computer')
     network_count = _count_by_type('network_device')
@@ -99,6 +116,22 @@ def get_dashboard():
         'totalmeasuringtool': measuringtool_count,
         'activeassets': status_dict.get('In Use', 0),
         'inrepair': status_dict.get('In Repair', 0),
+        # What the totals actually cover, so the page can say so rather than
+        # leaving a reader to work it out from a number that surprised them.
+        # Every count here filters isactive, and the dual-path collapse is a
+        # per-site toggle - asserting it unconditionally would be false at a
+        # site that has it off.
+        'countsactiveonly': True,
+        'dualpathcollapsed': bool(dualpath_single_machine_enabled()),
+        # Every status, not just the two that had callers. The counts were
+        # already built for the rollup above and then thrown away, so a reader
+        # could see "In Use" and had no way to learn where the rest went.
+        'statuses': status_dict,
+        # A DIFFERENT thing to a status, and the one that makes a total
+        # disagree with a list page. isactive is the soft-delete flag: these
+        # rows are excluded from every count above, so they are in no status
+        # bucket and cannot be found by adding the buckets up.
+        'inactiveexcluded': _inactive_excluded(),
         # Structured data
         'counts': {
             'machines': machine_count,

shopdb/core/api/scripts.py

@@ -0,0 +1,206 @@
+"""Script library API endpoints.
+
+The ShopDB-hosted script library backs the endpoint run_script delivery path: a
+PowerShell script is authored/uploaded once here, and the endpoint agent fetches
+its body by digest on push (the same model as application installer delivery).
+The body never crosses the command wire and is never pre-staged on the endpoint;
+the agent downloads it with its collector credential and verifies sha256.
+
+Gated on the applications.* permissions - scripts are software delivery like
+apps, so they ride the same operator grant rather than a separate permission.
+"""
+
+import os
+
+from flask import Blueprint, request, current_app
+
+from shopdb.extensions import db
+from shopdb.core.models import Script, RUNCONTEXTS, RUNCONTEXT_SYSTEM, AuditLog
+from shopdb.utils.responses import (
+    success_response,
+    error_response,
+    ErrorCodes,
+)
+from shopdb.utils.authz import require_permission
+from shopdb.utils.uploads import send_upload
+from shopdb.core.api.applications import _sha256_of, _replace_upload
+
+# Scripts are PowerShell for the endpoint agent. The guard is a sanity check on
+# the upload, not a security boundary - the download route always sends as an
+# attachment and the agent runs it explicitly.
+SCRIPT_EXTENSIONS = {'.ps1'}
+SCRIPT_URL_PREFIX = '/api/scripts/body/'
+# A delivery script is small; cap well under the installer limit.
+MAX_SCRIPT_BYTES = 1 * 1024 * 1024
+
+
+def _script_dir():
+    return os.path.join(current_app.instance_path, 'scripts')
+
+
+scripts_bp = Blueprint('scripts', __name__)
+
+
+@scripts_bp.route('', methods=['GET'])
+@require_permission('applications.view')
+def list_scripts():
+    """List scripts (active by default)."""
+    query = Script.query
+    if request.args.get('active', 'true').lower() != 'false':
+        query = query.filter(Script.isactive == True)  # noqa: E712
+    if search := request.args.get('search'):
+        query = query.filter(Script.scriptname.ilike(f'%{search}%'))
+    rows = query.order_by(Script.scriptname).all()
+    return success_response([row.to_dict() for row in rows])
+
+
+@scripts_bp.route('/<int:script_id>', methods=['GET'])
+@require_permission('applications.view')
+def get_script(script_id: int):
+    script = db.session.get(Script, script_id)
+    if not script:
+        return error_response(ErrorCodes.NOT_FOUND, 'Script not found', http_code=404)
+    return success_response(script.to_dict())
+
+
+def _validate_runcontext(value):
+    """Normalize/validate a runcontext, defaulting to system when absent."""
+    if value is None:
+        return RUNCONTEXT_SYSTEM, None
+    value = str(value).strip().lower()
+    if value not in RUNCONTEXTS:
+        return None, 'runcontext must be one of: {}'.format(', '.join(sorted(RUNCONTEXTS)))
+    return value, None
+
+
+@scripts_bp.route('', methods=['POST'])
+@require_permission('applications.create')
+def create_script():
+    """Create a script catalog entry (body uploaded separately)."""
+    data = request.get_json(silent=True) or {}
+    name = (data.get('scriptname') or '').strip()
+    if not name:
+        return error_response(ErrorCodes.VALIDATION_ERROR, 'scriptname is required')
+    if Script.query.filter_by(scriptname=name).first():
+        return error_response(ErrorCodes.VALIDATION_ERROR,
+                              f'A script named {name!r} already exists')
+    runcontext, err = _validate_runcontext(data.get('runcontext'))
+    if err:
+        return error_response(ErrorCodes.VALIDATION_ERROR, err)
+
+    script = Script(
+        scriptname=name,
+        scriptdescription=data.get('scriptdescription'),
+        runcontext=runcontext,
+    )
+    db.session.add(script)
+    db.session.commit()
+    AuditLog.log('created', 'Script', entityid=script.scriptid,
+                 entityname=name,
+                 changes={'runcontext': {'new': runcontext}})
+    db.session.commit()
+    return success_response(script.to_dict(), message='Script created', http_code=201)
+
+
+@scripts_bp.route('/<int:script_id>', methods=['PUT'])
+@require_permission('applications.edit')
+def update_script(script_id: int):
+    script = db.session.get(Script, script_id)
+    if not script:
+        return error_response(ErrorCodes.NOT_FOUND, 'Script not found', http_code=404)
+    data = request.get_json(silent=True) or {}
+
+    if 'scriptname' in data:
+        name = (data.get('scriptname') or '').strip()
+        if not name:
+            return error_response(ErrorCodes.VALIDATION_ERROR, 'scriptname cannot be empty')
+        clash = Script.query.filter(Script.scriptname == name,
+                                    Script.scriptid != script_id).first()
+        if clash:
+            return error_response(ErrorCodes.VALIDATION_ERROR,
+                                  f'A script named {name!r} already exists')
+        script.scriptname = name
+    if 'scriptdescription' in data:
+        script.scriptdescription = data.get('scriptdescription')
+    if 'runcontext' in data:
+        runcontext, err = _validate_runcontext(data.get('runcontext'))
+        if err:
+            return error_response(ErrorCodes.VALIDATION_ERROR, err)
+        script.runcontext = runcontext
+    if 'isactive' in data:
+        script.isactive = bool(data.get('isactive'))
+
+    db.session.commit()
+    return success_response(script.to_dict(), message='Script updated')
+
+
+@scripts_bp.route('/<int:script_id>', methods=['DELETE'])
+@require_permission('applications.delete')
+def delete_script(script_id: int):
+    script = db.session.get(Script, script_id)
+    if not script:
+        return error_response(ErrorCodes.NOT_FOUND, 'Script not found', http_code=404)
+    name = script.scriptname
+    db.session.delete(script)
+    db.session.commit()
+    AuditLog.log('deleted', 'Script', entityid=script_id, entityname=name)
+    db.session.commit()
+    return success_response(None, message='Script deleted')
+
+
+@scripts_bp.route('/<int:script_id>/body', methods=['POST'])
+@require_permission('applications.edit')
+def upload_script_body(script_id: int):
+    """Upload (or replace) the .ps1 body for a script.
+
+    multipart/form-data: file=<script.ps1>. Stored as script-<id>.ps1 and the
+    digest is stamped so the endpoint delivery path can pin exact bytes.
+    """
+    script = db.session.get(Script, script_id)
+    if not script:
+        return error_response(ErrorCodes.NOT_FOUND, 'Script not found', http_code=404)
+
+    upload = request.files.get('file')
+    if not upload or not upload.filename:
+        return error_response(ErrorCodes.VALIDATION_ERROR, 'No file provided')
+    ext = os.path.splitext(upload.filename)[1].lower()
+    if ext not in SCRIPT_EXTENSIONS:
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'Unsupported script type {}. Allowed: {}'.format(
+                ext, ', '.join(sorted(SCRIPT_EXTENSIONS))))
+
+    upload.stream.seek(0, os.SEEK_END)
+    size = upload.stream.tell()
+    upload.stream.seek(0)
+    if size > MAX_SCRIPT_BYTES:
+        return error_response(
+            ErrorCodes.VALIDATION_ERROR,
+            'Script is {:.0f}KB; the limit is {:.0f}KB.'.format(
+                size / 1024, MAX_SCRIPT_BYTES / 1024))
+
+    filename = _replace_upload(_script_dir(), f'script-{script_id}', ext, upload)
+    script.filename = filename
+    script.sha256 = _sha256_of(upload.stream)
+    db.session.commit()
+
+    AuditLog.log('updated', 'Script', entityid=script_id,
+                 entityname=script.scriptname,
+                 changes={'filename': {'new': filename},
+                          'sha256': {'new': script.sha256},
+                          'bytes': {'new': size}})
+    db.session.commit()
+    return success_response(
+        {**script.to_dict(), 'uploadedfilename': upload.filename, 'bytes': size},
+        message='Script body uploaded')
+
+
+@scripts_bp.route('/body/<path:filename>', methods=['GET'])
+@require_permission('applications.view')
+def download_script_body(filename: str):
+    """Operator-facing script body download (JWT/RBAC).
+
+    The endpoint agent uses the collector-authenticated plugin route instead;
+    this one is for humans reviewing what a script does.
+    """
+    return send_upload(_script_dir(), filename, as_attachment=True)

shopdb/core/models/__init__.py

@@ -16,6 +16,8 @@ from .communication import Communication, CommunicationType
 from .user import (User, Role, Permission, full_permission_catalog,
                    guest_grantable_permissions)
 from .application import Application, AppVersion
+from .script_library import (Script, RUNCONTEXTS, RUNCONTEXT_SYSTEM,
+                             RUNCONTEXT_USER)
 from .supportteam import SupportTeam, SupportTeamContact
 from .setting import Setting
 from .auditlog import AuditLog
@@ -62,6 +64,11 @@ __all__ = [
     # Applications
     'Application',
     'AppVersion',
+    # Script library (endpoint run_script delivery)
+    'Script',
+    'RUNCONTEXTS',
+    'RUNCONTEXT_SYSTEM',
+    'RUNCONTEXT_USER',
     # Support teams
     'SupportTeam',
     'SupportTeamContact',

shopdb/core/models/application.py

@@ -34,6 +34,11 @@ class Application(BaseModel):
         comment='Required on all PCs (drives the software-compliance report)'
     )
     image = db.Column(db.String(255))
+    # Digest of the app-level uploaded installer (installpath). Lets the
+    # endpoint update_app push resolve an install source by app alone, with no
+    # AppVersion row: the agent fetches the pinned bytes and verifies this hash.
+    # Stamped on package upload; null when installpath is a site share path.
+    installersha256 = db.Column(db.String(64))
 
     # Relationships
     supportteam = db.relationship('SupportTeam', back_populates='applications')
@@ -70,6 +75,12 @@ class AppVersion(db.Model):
     notes = db.Column(db.String(255))
     dateadded = db.Column(db.DateTime, default=_utcnow)
     isactive = db.Column(db.Boolean, default=True)
+    # Delivery metadata for the endpoint installer push. sha256 pins the exact
+    # bytes the agent must fetch (warn-not-block for now; hardened later);
+    # silentargs carries this version's unattended-install switches because
+    # homegrown EXEs vary by developer (MSI is uniform via msiexec).
+    sha256 = db.Column(db.String(64))
+    silentargs = db.Column(db.String(255))
 
     # Relationships
     application = db.relationship('Application', back_populates='versions')
@@ -88,7 +99,9 @@ class AppVersion(db.Model):
             'releasedate': self.releasedate.isoformat() if self.releasedate else None,
             'notes': self.notes,
             'dateadded': self.dateadded.isoformat() + 'Z' if self.dateadded else None,
-            'isactive': self.isactive
+            'isactive': self.isactive,
+            'sha256': self.sha256,
+            'silentargs': self.silentargs
         }
 
     def __repr__(self):

shopdb/core/models/script_library.py

@@ -0,0 +1,43 @@
+"""ShopDB-hosted script library for endpoint run_script delivery.
+
+A script is authored/uploaded once here, and the endpoint agent fetches its
+body by digest on push (mirroring the application installer delivery path). The
+body never lives on the endpoint ahead of time and never crosses the command
+wire; the agent downloads it with its collector credential and verifies sha256.
+"""
+
+from shopdb.extensions import db
+from .base import BaseModel
+
+# Execution context for a delivered script. SYSTEM is the service account the
+# agent already runs as (no user session, cannot see mapped drives); USER runs
+# in the interactive logged-on user's session. Default SYSTEM stays fail-safe.
+RUNCONTEXT_SYSTEM = 'system'
+RUNCONTEXT_USER = 'user'
+RUNCONTEXTS = (RUNCONTEXT_SYSTEM, RUNCONTEXT_USER)
+
+
+class Script(BaseModel):
+    """ShopDB-hosted PowerShell script the endpoint agent runs on push."""
+    __tablename__ = 'scripts'
+
+    scriptid = db.Column(db.Integer, primary_key=True)
+    scriptname = db.Column(db.String(100), unique=True, nullable=False)
+    scriptdescription = db.Column(db.String(255))
+    # Stored file name under the scripts upload directory (script-<id>.ps1). The
+    # body is served by the collector-authenticated download route.
+    filename = db.Column(db.String(255))
+    # Digest of the stored body; the agent verifies this before running.
+    sha256 = db.Column(db.String(64))
+    # 'system' or 'user' (see RUNCONTEXTS). Per-script so a drive-mapping script
+    # can target the user session while a machine-wide script runs as SYSTEM.
+    runcontext = db.Column(db.String(16), default=RUNCONTEXT_SYSTEM, nullable=False)
+
+    def to_dict(self):
+        result = super().to_dict()
+        # hasbody lets the UI disable pushing a script with no uploaded file.
+        result['hasbody'] = bool(self.filename and self.sha256)
+        return result
+
+    def __repr__(self):
+        return f"<Script {self.scriptname}>"