Files
cproudlock 838932a72d geenforce: gate publishing on the library version, not on the manifest's own
The publish gate exists because a minor version bump that NARROWS behaviour is
not backward compatible: _CmmVersion arrived in lib 2.6, and an older lib does
not know the field, so every gated entry looks unfiltered and it installs every
PC-DMIS version it cannot detect, on every CMM, in one cycle.

It was comparing the fleet's reported library versions against manifestversion.
That is the manifest's own 'Version' field. For a share-imported manifest the
two numbering schemes happen to coincide; for a scope authored in code they do
not, and seed_display_scope writes '2.0' - which every kiosk exceeds. So the
gate passed on the scope that most needed it.

A scope now declares minlibversion. Unset, the requirement is DERIVED from what
the manifest actually uses, so a scope written before this column existed is
still judged on its contents rather than on a number that says nothing about the
library. Only features that narrow behaviour belong in that table; an addition
an old lib ignores harmlessly needs no floor. manifestversion remains the last
fallback, which preserves what share-imported manifests already relied on.
2026-08-14 13:47:11 -04:00

575 lines
22 KiB
Python

"""DB-touching operations for the manifest store.
Kept out of the CLI and routes so both share one implementation:
- `replace_scope_draft` imports/re-imports a scope's DRAFT entries WITHOUT
touching its published-version history (re-import is idempotent + safe).
- `publish_scope` freezes the current draft into a new immutable snapshot.
- `rollback_scope` / `export_scope_to_share` round out the publish lifecycle.
"""
import hashlib
import json
import os
import tempfile
from datetime import datetime, timezone
from flask import current_app
from sqlalchemy import func
from shopdb.api import db, Application, normalize_display_role
from .models import (
ManifestScope, ManifestPublishedVersion, ManifestEnforcementReport,
ManifestEnforcementResult, ManifestPayload, ManifestBlob,
)
from .importer import build_entry
from .serializer import scope_to_json
class LibVersionTooOldError(Exception):
"""Publishing this scope would reach PCs whose enforcer lib is too old.
Carries the detail a human needs to act: what the manifest requires, the
lowest lib version the fleet reports for this scope, and which hosts are
behind.
"""
def __init__(self, scopename, required, floor, hosts):
self.scopename = scopename
self.required = required
self.floor = floor
self.hosts = hosts
shown = ', '.join(hosts[:10])
if len(hosts) > 10:
shown += f' (+{len(hosts) - 10} more)'
super().__init__(
f'{scopename} manifest is version {required} but {len(hosts)} '
f'reporting PC(s) run enforcer lib {floor}: {shown}. '
'Push the newer lib to the fleet first, or publish with force=True '
'if you are certain those PCs must not receive this scope.')
def _canonical_subtype(value):
"""Store a reported subtype under core's role vocabulary where it names one.
A display reports the literal contents of C:\\Enrollment\\display-type.txt,
hand-edited on the machine, so casing drifts freely. Normalising on the way
in means the fleet table reads one vocabulary rather than whatever each
kiosk happened to be typed as.
A value that names NO known role is kept VERBATIM, not dropped: an unknown
subtype is a kiosk with a typo or a role nobody told the server about, and
both are things you want to see in the table rather than have silently
blanked. Non-display scopes report no subtype at all and land as None.
"""
text = (value or '').strip()
if not text:
return None
return normalize_display_role(text) or text
def _utcnow():
return datetime.now(timezone.utc).replace(tzinfo=None)
def parse_libversion(text):
"""'2.6' -> (2, 6). Returns None for missing or unparseable values.
None means "cannot tell", and every caller treats that as "do not claim
this PC is safe" rather than "this PC is fine" - an unreadable version is
exactly the case where guessing is expensive.
"""
if not text:
return None
parts = str(text).strip().split('.')
try:
return (int(parts[0]), int(parts[1]) if len(parts) > 1 else 0)
except (ValueError, IndexError):
return None
def hosts_below_libversion(scopename, phase, required):
"""Hosts whose CURRENT report for this scope runs a lib older than required.
A report with no/unparseable enforcerversion counts as behind: the field was
added with the summary-emitting engine, so its absence means an old lib.
Returns (hosts, floor) where floor is the lowest parseable version string
seen, or None when nothing reported.
"""
wanted = parse_libversion(required)
if not wanted:
return [], None
reports = ManifestEnforcementReport.query.filter_by(
scopename=scopename, phase=phase, iscurrent=True).all()
behind = []
lowest = None
for report in reports:
actual = parse_libversion(report.enforcerversion)
if actual is None or actual < wanted:
behind.append(report.hostname)
if actual is not None and (lowest is None or actual < lowest):
lowest = actual
floor = f'{lowest[0]}.{lowest[1]}' if lowest else None
return sorted(set(behind)), floor
# Manifest features and the client library that first understood them. A
# library older than this does not merely skip the feature, it MISREADS the
# entry: 2.6 added _CmmVersion, and an older lib treats a version-gated entry as
# ungated and installs every PC-DMIS build it cannot detect.
#
# Only features that NARROW behaviour belong here. An addition an old lib
# ignores harmlessly does not need a floor.
LIBVERSION_FEATURES = (
('cmmversion', '2.6'),
)
def required_libversion(scope):
"""Lowest client library that may enforce this scope.
An explicit minlibversion wins. Otherwise it is derived from the features
the draft actually uses, so a scope written before this column existed is
still gated on what it contains rather than on manifestversion - which is
the manifest's own 'Version' field and says nothing about the library.
Falls back to manifestversion when nothing else applies, preserving the
behaviour share-imported manifests already relied on, where the two
numbering schemes do coincide.
"""
declared = (scope.minlibversion or '').strip()
if declared:
return declared
floor = None
for attribute, version in LIBVERSION_FEATURES:
if any(getattr(entry, attribute, None) for entry in scope.entries):
candidate = parse_libversion(version)
if candidate and (floor is None or candidate > floor[0]):
floor = (candidate, version)
if floor:
return floor[1]
return scope.manifestversion
def _installed_app_model():
"""Lazily import the computers plugin's ComputerInstalledApp.
The computers plugin is optional; importing it lazily (not at module load)
keeps geenforce usable when computers is absent or disabled. Mirrors
collector._computer_models. Returns the model class, or None if absent.
"""
try:
from plugins.computers.models import ComputerInstalledApp
return ComputerInstalledApp
except ImportError:
return None
def replace_scope_draft(scopename, phase, manifest):
"""Create/refresh a scope and REPLACE its draft entries. Published versions
are left untouched. Returns the scope (uncommitted)."""
scope = ManifestScope.query.filter_by(
scopename=scopename, phase=phase).first()
if not scope:
scope = ManifestScope(scopename=scopename, phase=phase)
db.session.add(scope)
scope.manifestversion = str(manifest.get('Version', '1.0'))
scope.topcomment = manifest.get('_comment')
scope.site = manifest.get('Site')
scope.iscommon = (scopename == 'common')
# Clear existing draft entries via the delete-orphan cascade. This also
# EMPTIES the in-memory collection, so a later lookup on scope.entries cannot
# match a stale (deleted) entry. A per-object db.session.delete() leaves the
# deleted objects in scope.entries until expiry, which on a re-publish made a
# caller (seed_display_scope) pick an old deleted entry and attach a payload
# to its dead entryid -> MySQL FK 1452 (SQLite does not enforce it, so the
# idempotency test missed it).
scope.entries.clear()
db.session.flush()
for i, entry_dict in enumerate(manifest.get('Applications') or []):
scope.entries.append(build_entry(entry_dict, i))
return scope
def publish_scope(scopename, phase, notes=None, publishedby=None, force=False):
"""Freeze the current draft into a new published snapshot. Returns the
version number (uncommitted).
Refuses when the manifest is newer than the enforcer lib on PCs that report
for this scope, unless force=True.
WHY THIS GATE EXISTS: the lib treats a minor-newer manifest as backward
compatible and carries on. That holds for additions that WIDEN behaviour (a
new Type is skipped by an old lib), and inverts for one that NARROWS it. The
_CmmVersion filter arrived in lib 2.6 as a minor bump: an older lib does not
know the field, so every gated entry looks unfiltered and it installs EVERY
PC-DMIS version it cannot detect, on every CMM, within one cycle. The share
runbook says push the lib first; a runbook is not a control. This is, and it
reads the version the fleet actually reports rather than the one someone
believes is deployed.
A scope nobody has reported for yet cannot be checked, so it publishes: a
fresh site would otherwise be unable to publish anything at all.
"""
scope = ManifestScope.query.filter_by(
scopename=scopename, phase=phase).first()
if not scope:
raise ValueError(f'No scope {scopename}/{phase}')
if not force:
required = required_libversion(scope)
behind, floor = hosts_below_libversion(scopename, phase, required)
if behind:
raise LibVersionTooOldError(
scopename, required, floor or 'unknown', behind)
text = scope_to_json(scope)
maxversion = db.session.query(
func.max(ManifestPublishedVersion.versionnumber)
).filter_by(scopeid=scope.scopeid).scalar() or 0
ManifestPublishedVersion.query.filter_by(
scopeid=scope.scopeid, iscurrent=True
).update({'iscurrent': False})
db.session.add(ManifestPublishedVersion(
scopeid=scope.scopeid,
versionnumber=maxversion + 1,
manifestjson=text,
publishedat=_utcnow(),
publishedby=publishedby,
iscurrent=True,
notes=notes))
return maxversion + 1
def rollback_scope(scopename, phase, versionnumber):
"""Make an older published version current again (uncommitted)."""
scope = ManifestScope.query.filter_by(
scopename=scopename, phase=phase).first()
if not scope:
raise ValueError(f'No scope {scopename}/{phase}')
target = ManifestPublishedVersion.query.filter_by(
scopeid=scope.scopeid, versionnumber=versionnumber).first()
if not target:
raise ValueError(f'No version {versionnumber} for {scopename}')
ManifestPublishedVersion.query.filter_by(
scopeid=scope.scopeid, iscurrent=True
).update({'iscurrent': False})
target.iscurrent = True
return versionnumber
def record_enforcement_report(payload):
"""Record one PC's enforcement cycle (observed state). Upserts the latest
report per (hostname, scopename, phase) and keeps prior ones as history.
Payload (all but hostname optional):
hostname, scopename, phase, appliedversion, enforcerversion, lastcheckin,
counts {installed, skipped, failed, filtered},
results [{name, action, selfhealed, exitcode, message}]
Returns the new ManifestEnforcementReport (uncommitted).
"""
hostname = (payload.get('hostname') or '').strip()
if not hostname:
raise ValueError('hostname is required')
scopename = (payload.get('scopename') or '').strip()
phase = (payload.get('phase') or 'runtime').strip()
counts = payload.get('counts') or {}
results = payload.get('results') or []
# Guard wrong JSON shapes: a bad type must be a clean 400, not a 500 from a
# later .get()/iteration (the route maps ValueError to 400).
if not isinstance(counts, dict):
raise ValueError('counts must be an object')
if not isinstance(results, list):
raise ValueError('results must be a list')
failed = int(counts.get('failed', 0))
installed = int(counts.get('installed', 0))
# Derive status from EXPLICIT self-heal flags only, never the raw installed
# count: every healthy cycle runs Always/no-detection scripts (asset report,
# VNC firewall, EventSaver) that the engine counts as "installed", so keying
# self-heal off installed>0 would mark the common scope selfhealed forever
# and make 'ok' unreachable. A drift correction is one the client flags.
if failed > 0:
status = 'failed'
elif any(r.get('selfhealed') for r in results):
status = 'selfhealed'
else:
status = 'ok'
# Demote the prior current report for this host+scope. Hostname match is
# case-insensitive to match the ilike read path - otherwise a PC reporting
# its name in different casing would leave two iscurrent rows.
ManifestEnforcementReport.query.filter(
func.lower(ManifestEnforcementReport.hostname) == hostname.lower(),
ManifestEnforcementReport.scopename == scopename,
ManifestEnforcementReport.phase == phase,
ManifestEnforcementReport.iscurrent == True
).update({'iscurrent': False}, synchronize_session=False)
report = ManifestEnforcementReport(
hostname=hostname,
scopename=scopename,
phase=phase,
appliedversion=payload.get('appliedversion'),
enforcerversion=payload.get('enforcerversion'),
subtype=_canonical_subtype(payload.get('subtype')),
installedcount=installed,
skippedcount=int(counts.get('skipped', 0)),
failedcount=failed,
filteredcount=int(counts.get('filtered', 0)),
status=status,
lastcheckin=_parse_dt(payload.get('lastcheckin')),
receivedat=_utcnow(),
iscurrent=True)
for item in results:
report.results.append(ManifestEnforcementResult(
entryname=item.get('name', ''),
action=item.get('action', ''),
# Explicit flag only; do NOT infer from action == 'installed'
# (Always/no-detection scripts install every cycle without being a
# drift correction).
selfhealed=bool(item.get('selfhealed', False)),
exitcode=item.get('exitcode'),
message=item.get('message')))
db.session.add(report)
return report
def _parse_dt(value):
if not value:
return None
try:
return datetime.fromisoformat(str(value).replace('Z', '+00:00')).replace(
tzinfo=None)
except (ValueError, TypeError):
return None
def compliance_for_scope(scope):
"""Fleet-install coverage for a scope's app-linked entries.
One row per ManifestEntry that carries a curated appid (unlinked entries
are skipped), ordered by sortorder. Counts come from the computers plugin's
ComputerInstalledApp (active rows only, one row per PC per app). Degrades
gracefully with null counts when the computers plugin is absent.
Returns the response data dict (scopeid/scopename/phase/computersplugin/rows).
"""
linked = [e for e in sorted(scope.entries, key=lambda e: e.sortorder)
if e.appid is not None]
installedmodel = _installed_app_model()
computerspresent = installedmodel is not None
# One grouped query for the whole scope's appid set (fleet is tiny).
installedbyapp = {}
if computerspresent and linked:
appids = {e.appid for e in linked}
counted = db.session.query(
installedmodel.appid, func.count(installedmodel.id)
).filter(
installedmodel.isactive == True,
installedmodel.appid.in_(appids),
).group_by(installedmodel.appid).all()
installedbyapp = {appid: count for appid, count in counted}
rows = []
for entry in linked:
app = db.session.get(Application, entry.appid)
appname = app.appname if app else None
expectedversion = (entry.detectionvalue
if entry.detectionmethod == 'FileVersion' else None)
if not computerspresent:
installedcount = None
versionmatchcount = None
coveragenote = ('computers plugin not installed; install counts '
'unavailable')
else:
installedcount = installedbyapp.get(entry.appid, 0)
if installedcount == 0:
versionmatchcount = None if expectedversion is None else 0
coveragenote = 'not installed on any collected PC'
elif expectedversion is None:
versionmatchcount = None
method = entry.detectionmethod or 'none'
coveragenote = (f'installed on {installedcount} PC(s); no '
f'version target (detection is {method})')
else:
versionmatchcount = db.session.query(
func.count(installedmodel.id)
).filter(
installedmodel.isactive == True,
installedmodel.appid == entry.appid,
installedmodel.installedversion == expectedversion,
).scalar() or 0
coveragenote = (f'{versionmatchcount} of {installedcount} '
'collected PCs on the expected version')
rows.append({
'entryid': entry.entryid,
'entryname': entry.name,
'appid': entry.appid,
'appname': appname,
'expectedversion': expectedversion,
'installedcount': installedcount,
'versionmatchcount': versionmatchcount,
'coveragenote': coveragenote,
})
return {
'scopeid': scope.scopeid,
'scopename': scope.scopename,
'phase': scope.phase,
'computersplugin': computerspresent,
'rows': rows,
}
def _payload_dir():
d = os.path.join(current_app.instance_path, 'geenforce', 'payloads')
os.makedirs(d, exist_ok=True)
return d
def blob_path(sha256):
"""Filesystem path where a blob's bytes live (may or may not exist)."""
return os.path.join(_payload_dir(), sha256)
def blob_referenced_by_scopes(sha256, scopenames):
"""True when any of these scopes' CURRENT published manifest references
sha256 as an entry payload.
Backs the resource-bound token check on GET /payload: a token pinned to its
own scope(s) may only pull blobs those scopes actually ship, not any blob by
hash. Empty scopenames -> False (a bound-but-empty token reaches nothing).
"""
if not scopenames:
return False
rows = db.session.query(ManifestPublishedVersion).join(
ManifestScope,
ManifestPublishedVersion.scopeid == ManifestScope.scopeid,
).filter(
ManifestScope.scopename.in_(list(scopenames)),
ManifestPublishedVersion.iscurrent == True, # noqa: E712
).all()
for row in rows:
try:
doc = json.loads(row.manifestjson)
except (ValueError, TypeError):
continue
for entry in doc.get('Applications', []) or []:
if entry.get('PayloadSha256') == sha256:
return True
return False
def store_blob(rawbytes, filename, contenttype=None):
"""Store bytes in the content-addressed payload store; return the sha256.
Deduped by content hash - an already-present blob is not rewritten. Upserts
a ManifestBlob registry row (uncommitted). Entries reference it by
payloadsha256; the client fetches GET /api/geenforce/payload/<sha256>.
"""
sha = hashlib.sha256(rawbytes).hexdigest()
path = blob_path(sha)
if not os.path.exists(path):
fd, tmp = tempfile.mkstemp(dir=_payload_dir(), suffix='.tmp')
try:
with os.fdopen(fd, 'wb') as handle:
handle.write(rawbytes)
os.replace(tmp, path)
except Exception:
if os.path.exists(tmp):
os.remove(tmp)
raise
if not db.session.get(ManifestBlob, sha):
db.session.add(ManifestBlob(
sha256=sha, filename=filename, contenttype=contenttype,
sizebytes=len(rawbytes), createdat=_utcnow()))
return sha
def store_inline_payload(entry, filename, contenttype, rawbytes):
"""Replace the entry's inline payload with these bytes (uncommitted).
Computes payloadsha256, upserts the single ManifestPayload row (the table
has no unique constraint on entryid, so the one-inline-payload-per-entry
rule is enforced here), and points the entry at it (payloadsource='inline').
Returns the ManifestPayload.
"""
sha = hashlib.sha256(rawbytes).hexdigest()
# Flush any pending inserts (e.g. a prior entry's inline payload) BEFORE the
# bulk delete. Otherwise the delete's autoflush interleaves that half-built
# INSERT and, on MySQL, fails the manifestpayloads->manifestentries FK
# (SQLite does not enforce it the same way, so tests miss this). Then delete
# existing rows for this entry with autoflush off + no session-sync.
db.session.flush()
with db.session.no_autoflush:
ManifestPayload.query.filter_by(
entryid=entry.entryid).delete(synchronize_session=False)
db.session.flush()
payload = ManifestPayload(
entryid=entry.entryid,
filename=filename,
contenttype=contenttype,
payloadbytes=rawbytes,
payloadsha256=sha,
uploadedat=_utcnow())
db.session.add(payload)
entry.payloadsource = 'inline'
entry.payloadref = filename
entry.payloadsha256 = sha
return payload
def export_scope_to_share(scopename, phase, shareroot):
"""Write a scope's current published JSON to the share, backing up the old
file to _meta/history first. Returns the written path."""
scope = ManifestScope.query.filter_by(
scopename=scopename, phase=phase).first()
if not scope:
raise ValueError(f'No scope {scopename}/{phase}')
published = scope.publishedversions.filter_by(iscurrent=True).first()
if not published:
raise ValueError(f'{scopename} has no published version')
if phase == 'preinstall':
target = os.path.join(shareroot, 'preinstall.json')
else:
target = os.path.join(shareroot, scopename, 'manifest.json')
os.makedirs(os.path.dirname(target), exist_ok=True)
if os.path.isfile(target):
historydir = os.path.join(shareroot, '_meta', 'history')
os.makedirs(historydir, exist_ok=True)
stamp = _utcnow().strftime('%Y%m%d-%H%M%S')
with open(target) as src:
old = src.read()
with open(os.path.join(historydir, f'{stamp}-{scopename}.json'), 'w') as dst:
dst.write(old)
# Atomic write: a partial/failed write must never leave the live on-share
# manifest (which every PC reads) truncated. Write a temp file in the same
# directory, then rename over the target.
targetdir = os.path.dirname(target)
fd, tmppath = tempfile.mkstemp(dir=targetdir, suffix='.tmp')
try:
with os.fdopen(fd, 'w') as handle:
handle.write(published.manifestjson)
os.replace(tmppath, target)
except Exception:
if os.path.exists(tmppath):
os.remove(tmppath)
raise
return target