9e34fafce5
The engine treats a minor-newer manifest as backward compatible and carries on. That holds for additions which WIDEN behaviour - an old lib skips a Type it does not know - and inverts for one that NARROWS it. _CmmVersion arrived in lib 2.6 as a minor bump, so a PC on 2.5 does not recognise the field, reads every gated entry as unfiltered, and installs every PC-DMIS version it cannot detect, on every CMM, within one cycle. The share runbook already says push the lib first. A runbook is not a control, and the failure is silent, fleet-wide and about five minutes fast. ShopDB already had the evidence and was not using it: every enforcement report carries the enforcer version, and publish_scope had no gate at all. It now compares the scope's manifest version against the versions PCs actually report for that scope and refuses when any is behind, naming the hosts. force=True for someone who knows why. A report with no or an unreadable version counts as behind - that field arrived with the summary-emitting engine, so its absence IS an old lib, and treating unknown as safe is precisely how this fails open. A scope nobody has reported for still publishes, or a fresh site could never publish anything. Versions compare numerically, since as text '2.10' sorts below '2.9'. Also exposed as a preflight endpoint so the UI can warn before someone clicks publish, and as a 409 with the offending hosts rather than a 500.
516 lines
20 KiB
Python
516 lines
20 KiB
Python
"""DB-touching operations for the manifest store.
|
|
|
|
Kept out of the CLI and routes so both share one implementation:
|
|
- `replace_scope_draft` imports/re-imports a scope's DRAFT entries WITHOUT
|
|
touching its published-version history (re-import is idempotent + safe).
|
|
- `publish_scope` freezes the current draft into a new immutable snapshot.
|
|
- `rollback_scope` / `export_scope_to_share` round out the publish lifecycle.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
|
|
from flask import current_app
|
|
from sqlalchemy import func
|
|
|
|
from shopdb.api import db, Application
|
|
|
|
from .models import (
|
|
ManifestScope, ManifestPublishedVersion, ManifestEnforcementReport,
|
|
ManifestEnforcementResult, ManifestPayload, ManifestBlob,
|
|
)
|
|
from .importer import build_entry
|
|
from .serializer import scope_to_json
|
|
|
|
|
|
class LibVersionTooOldError(Exception):
|
|
"""Publishing this scope would reach PCs whose enforcer lib is too old.
|
|
|
|
Carries the detail a human needs to act: what the manifest requires, the
|
|
lowest lib version the fleet reports for this scope, and which hosts are
|
|
behind.
|
|
"""
|
|
|
|
def __init__(self, scopename, required, floor, hosts):
|
|
self.scopename = scopename
|
|
self.required = required
|
|
self.floor = floor
|
|
self.hosts = hosts
|
|
shown = ', '.join(hosts[:10])
|
|
if len(hosts) > 10:
|
|
shown += f' (+{len(hosts) - 10} more)'
|
|
super().__init__(
|
|
f'{scopename} manifest is version {required} but {len(hosts)} '
|
|
f'reporting PC(s) run enforcer lib {floor}: {shown}. '
|
|
'Push the newer lib to the fleet first, or publish with force=True '
|
|
'if you are certain those PCs must not receive this scope.')
|
|
|
|
|
|
def _utcnow():
|
|
return datetime.now(timezone.utc).replace(tzinfo=None)
|
|
|
|
|
|
def parse_libversion(text):
|
|
"""'2.6' -> (2, 6). Returns None for missing or unparseable values.
|
|
|
|
None means "cannot tell", and every caller treats that as "do not claim
|
|
this PC is safe" rather than "this PC is fine" - an unreadable version is
|
|
exactly the case where guessing is expensive.
|
|
"""
|
|
if not text:
|
|
return None
|
|
parts = str(text).strip().split('.')
|
|
try:
|
|
return (int(parts[0]), int(parts[1]) if len(parts) > 1 else 0)
|
|
except (ValueError, IndexError):
|
|
return None
|
|
|
|
|
|
def hosts_below_libversion(scopename, phase, required):
|
|
"""Hosts whose CURRENT report for this scope runs a lib older than required.
|
|
|
|
A report with no/unparseable enforcerversion counts as behind: the field was
|
|
added with the summary-emitting engine, so its absence means an old lib.
|
|
|
|
Returns (hosts, floor) where floor is the lowest parseable version string
|
|
seen, or None when nothing reported.
|
|
"""
|
|
wanted = parse_libversion(required)
|
|
if not wanted:
|
|
return [], None
|
|
|
|
reports = ManifestEnforcementReport.query.filter_by(
|
|
scopename=scopename, phase=phase, iscurrent=True).all()
|
|
|
|
behind = []
|
|
lowest = None
|
|
for report in reports:
|
|
actual = parse_libversion(report.enforcerversion)
|
|
if actual is None or actual < wanted:
|
|
behind.append(report.hostname)
|
|
if actual is not None and (lowest is None or actual < lowest):
|
|
lowest = actual
|
|
floor = f'{lowest[0]}.{lowest[1]}' if lowest else None
|
|
return sorted(set(behind)), floor
|
|
|
|
|
|
def _installed_app_model():
|
|
"""Lazily import the computers plugin's ComputerInstalledApp.
|
|
|
|
The computers plugin is optional; importing it lazily (not at module load)
|
|
keeps geenforce usable when computers is absent or disabled. Mirrors
|
|
collector._computer_models. Returns the model class, or None if absent.
|
|
"""
|
|
try:
|
|
from plugins.computers.models import ComputerInstalledApp
|
|
return ComputerInstalledApp
|
|
except ImportError:
|
|
return None
|
|
|
|
|
|
def replace_scope_draft(scopename, phase, manifest):
|
|
"""Create/refresh a scope and REPLACE its draft entries. Published versions
|
|
are left untouched. Returns the scope (uncommitted)."""
|
|
scope = ManifestScope.query.filter_by(
|
|
scopename=scopename, phase=phase).first()
|
|
if not scope:
|
|
scope = ManifestScope(scopename=scopename, phase=phase)
|
|
db.session.add(scope)
|
|
scope.manifestversion = str(manifest.get('Version', '1.0'))
|
|
scope.topcomment = manifest.get('_comment')
|
|
scope.site = manifest.get('Site')
|
|
scope.iscommon = (scopename == 'common')
|
|
|
|
# Clear existing draft entries via the delete-orphan cascade. This also
|
|
# EMPTIES the in-memory collection, so a later lookup on scope.entries cannot
|
|
# match a stale (deleted) entry. A per-object db.session.delete() leaves the
|
|
# deleted objects in scope.entries until expiry, which on a re-publish made a
|
|
# caller (seed_display_scope) pick an old deleted entry and attach a payload
|
|
# to its dead entryid -> MySQL FK 1452 (SQLite does not enforce it, so the
|
|
# idempotency test missed it).
|
|
scope.entries.clear()
|
|
db.session.flush()
|
|
|
|
for i, entry_dict in enumerate(manifest.get('Applications') or []):
|
|
scope.entries.append(build_entry(entry_dict, i))
|
|
return scope
|
|
|
|
|
|
def publish_scope(scopename, phase, notes=None, publishedby=None, force=False):
|
|
"""Freeze the current draft into a new published snapshot. Returns the
|
|
version number (uncommitted).
|
|
|
|
Refuses when the manifest is newer than the enforcer lib on PCs that report
|
|
for this scope, unless force=True.
|
|
|
|
WHY THIS GATE EXISTS: the lib treats a minor-newer manifest as backward
|
|
compatible and carries on. That holds for additions that WIDEN behaviour (a
|
|
new Type is skipped by an old lib), and inverts for one that NARROWS it. The
|
|
_CmmVersion filter arrived in lib 2.6 as a minor bump: an older lib does not
|
|
know the field, so every gated entry looks unfiltered and it installs EVERY
|
|
PC-DMIS version it cannot detect, on every CMM, within one cycle. The share
|
|
runbook says push the lib first; a runbook is not a control. This is, and it
|
|
reads the version the fleet actually reports rather than the one someone
|
|
believes is deployed.
|
|
|
|
A scope nobody has reported for yet cannot be checked, so it publishes: a
|
|
fresh site would otherwise be unable to publish anything at all.
|
|
"""
|
|
scope = ManifestScope.query.filter_by(
|
|
scopename=scopename, phase=phase).first()
|
|
if not scope:
|
|
raise ValueError(f'No scope {scopename}/{phase}')
|
|
|
|
if not force:
|
|
behind, floor = hosts_below_libversion(
|
|
scopename, phase, scope.manifestversion)
|
|
if behind:
|
|
raise LibVersionTooOldError(
|
|
scopename, scope.manifestversion, floor or 'unknown', behind)
|
|
|
|
text = scope_to_json(scope)
|
|
maxversion = db.session.query(
|
|
func.max(ManifestPublishedVersion.versionnumber)
|
|
).filter_by(scopeid=scope.scopeid).scalar() or 0
|
|
|
|
ManifestPublishedVersion.query.filter_by(
|
|
scopeid=scope.scopeid, iscurrent=True
|
|
).update({'iscurrent': False})
|
|
|
|
db.session.add(ManifestPublishedVersion(
|
|
scopeid=scope.scopeid,
|
|
versionnumber=maxversion + 1,
|
|
manifestjson=text,
|
|
publishedat=_utcnow(),
|
|
publishedby=publishedby,
|
|
iscurrent=True,
|
|
notes=notes))
|
|
return maxversion + 1
|
|
|
|
|
|
def rollback_scope(scopename, phase, versionnumber):
|
|
"""Make an older published version current again (uncommitted)."""
|
|
scope = ManifestScope.query.filter_by(
|
|
scopename=scopename, phase=phase).first()
|
|
if not scope:
|
|
raise ValueError(f'No scope {scopename}/{phase}')
|
|
target = ManifestPublishedVersion.query.filter_by(
|
|
scopeid=scope.scopeid, versionnumber=versionnumber).first()
|
|
if not target:
|
|
raise ValueError(f'No version {versionnumber} for {scopename}')
|
|
ManifestPublishedVersion.query.filter_by(
|
|
scopeid=scope.scopeid, iscurrent=True
|
|
).update({'iscurrent': False})
|
|
target.iscurrent = True
|
|
return versionnumber
|
|
|
|
|
|
def record_enforcement_report(payload):
|
|
"""Record one PC's enforcement cycle (observed state). Upserts the latest
|
|
report per (hostname, scopename, phase) and keeps prior ones as history.
|
|
|
|
Payload (all but hostname optional):
|
|
hostname, scopename, phase, appliedversion, enforcerversion, lastcheckin,
|
|
counts {installed, skipped, failed, filtered},
|
|
results [{name, action, selfhealed, exitcode, message}]
|
|
|
|
Returns the new ManifestEnforcementReport (uncommitted).
|
|
"""
|
|
hostname = (payload.get('hostname') or '').strip()
|
|
if not hostname:
|
|
raise ValueError('hostname is required')
|
|
scopename = (payload.get('scopename') or '').strip()
|
|
phase = (payload.get('phase') or 'runtime').strip()
|
|
counts = payload.get('counts') or {}
|
|
results = payload.get('results') or []
|
|
# Guard wrong JSON shapes: a bad type must be a clean 400, not a 500 from a
|
|
# later .get()/iteration (the route maps ValueError to 400).
|
|
if not isinstance(counts, dict):
|
|
raise ValueError('counts must be an object')
|
|
if not isinstance(results, list):
|
|
raise ValueError('results must be a list')
|
|
|
|
failed = int(counts.get('failed', 0))
|
|
installed = int(counts.get('installed', 0))
|
|
# Derive status from EXPLICIT self-heal flags only, never the raw installed
|
|
# count: every healthy cycle runs Always/no-detection scripts (asset report,
|
|
# VNC firewall, EventSaver) that the engine counts as "installed", so keying
|
|
# self-heal off installed>0 would mark the common scope selfhealed forever
|
|
# and make 'ok' unreachable. A drift correction is one the client flags.
|
|
if failed > 0:
|
|
status = 'failed'
|
|
elif any(r.get('selfhealed') for r in results):
|
|
status = 'selfhealed'
|
|
else:
|
|
status = 'ok'
|
|
|
|
# Demote the prior current report for this host+scope. Hostname match is
|
|
# case-insensitive to match the ilike read path - otherwise a PC reporting
|
|
# its name in different casing would leave two iscurrent rows.
|
|
ManifestEnforcementReport.query.filter(
|
|
func.lower(ManifestEnforcementReport.hostname) == hostname.lower(),
|
|
ManifestEnforcementReport.scopename == scopename,
|
|
ManifestEnforcementReport.phase == phase,
|
|
ManifestEnforcementReport.iscurrent == True
|
|
).update({'iscurrent': False}, synchronize_session=False)
|
|
|
|
report = ManifestEnforcementReport(
|
|
hostname=hostname,
|
|
scopename=scopename,
|
|
phase=phase,
|
|
appliedversion=payload.get('appliedversion'),
|
|
enforcerversion=payload.get('enforcerversion'),
|
|
installedcount=installed,
|
|
skippedcount=int(counts.get('skipped', 0)),
|
|
failedcount=failed,
|
|
filteredcount=int(counts.get('filtered', 0)),
|
|
status=status,
|
|
lastcheckin=_parse_dt(payload.get('lastcheckin')),
|
|
receivedat=_utcnow(),
|
|
iscurrent=True)
|
|
for item in results:
|
|
report.results.append(ManifestEnforcementResult(
|
|
entryname=item.get('name', ''),
|
|
action=item.get('action', ''),
|
|
# Explicit flag only; do NOT infer from action == 'installed'
|
|
# (Always/no-detection scripts install every cycle without being a
|
|
# drift correction).
|
|
selfhealed=bool(item.get('selfhealed', False)),
|
|
exitcode=item.get('exitcode'),
|
|
message=item.get('message')))
|
|
db.session.add(report)
|
|
return report
|
|
|
|
|
|
def _parse_dt(value):
|
|
if not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(str(value).replace('Z', '+00:00')).replace(
|
|
tzinfo=None)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def compliance_for_scope(scope):
|
|
"""Fleet-install coverage for a scope's app-linked entries.
|
|
|
|
One row per ManifestEntry that carries a curated appid (unlinked entries
|
|
are skipped), ordered by sortorder. Counts come from the computers plugin's
|
|
ComputerInstalledApp (active rows only, one row per PC per app). Degrades
|
|
gracefully with null counts when the computers plugin is absent.
|
|
|
|
Returns the response data dict (scopeid/scopename/phase/computersplugin/rows).
|
|
"""
|
|
linked = [e for e in sorted(scope.entries, key=lambda e: e.sortorder)
|
|
if e.appid is not None]
|
|
installedmodel = _installed_app_model()
|
|
computerspresent = installedmodel is not None
|
|
|
|
# One grouped query for the whole scope's appid set (fleet is tiny).
|
|
installedbyapp = {}
|
|
if computerspresent and linked:
|
|
appids = {e.appid for e in linked}
|
|
counted = db.session.query(
|
|
installedmodel.appid, func.count(installedmodel.id)
|
|
).filter(
|
|
installedmodel.isactive == True,
|
|
installedmodel.appid.in_(appids),
|
|
).group_by(installedmodel.appid).all()
|
|
installedbyapp = {appid: count for appid, count in counted}
|
|
|
|
rows = []
|
|
for entry in linked:
|
|
app = db.session.get(Application, entry.appid)
|
|
appname = app.appname if app else None
|
|
expectedversion = (entry.detectionvalue
|
|
if entry.detectionmethod == 'FileVersion' else None)
|
|
|
|
if not computerspresent:
|
|
installedcount = None
|
|
versionmatchcount = None
|
|
coveragenote = ('computers plugin not installed; install counts '
|
|
'unavailable')
|
|
else:
|
|
installedcount = installedbyapp.get(entry.appid, 0)
|
|
if installedcount == 0:
|
|
versionmatchcount = None if expectedversion is None else 0
|
|
coveragenote = 'not installed on any collected PC'
|
|
elif expectedversion is None:
|
|
versionmatchcount = None
|
|
method = entry.detectionmethod or 'none'
|
|
coveragenote = (f'installed on {installedcount} PC(s); no '
|
|
f'version target (detection is {method})')
|
|
else:
|
|
versionmatchcount = db.session.query(
|
|
func.count(installedmodel.id)
|
|
).filter(
|
|
installedmodel.isactive == True,
|
|
installedmodel.appid == entry.appid,
|
|
installedmodel.installedversion == expectedversion,
|
|
).scalar() or 0
|
|
coveragenote = (f'{versionmatchcount} of {installedcount} '
|
|
'collected PCs on the expected version')
|
|
|
|
rows.append({
|
|
'entryid': entry.entryid,
|
|
'entryname': entry.name,
|
|
'appid': entry.appid,
|
|
'appname': appname,
|
|
'expectedversion': expectedversion,
|
|
'installedcount': installedcount,
|
|
'versionmatchcount': versionmatchcount,
|
|
'coveragenote': coveragenote,
|
|
})
|
|
|
|
return {
|
|
'scopeid': scope.scopeid,
|
|
'scopename': scope.scopename,
|
|
'phase': scope.phase,
|
|
'computersplugin': computerspresent,
|
|
'rows': rows,
|
|
}
|
|
|
|
|
|
def _payload_dir():
|
|
d = os.path.join(current_app.instance_path, 'geenforce', 'payloads')
|
|
os.makedirs(d, exist_ok=True)
|
|
return d
|
|
|
|
|
|
def blob_path(sha256):
|
|
"""Filesystem path where a blob's bytes live (may or may not exist)."""
|
|
return os.path.join(_payload_dir(), sha256)
|
|
|
|
|
|
def blob_referenced_by_scopes(sha256, scopenames):
|
|
"""True when any of these scopes' CURRENT published manifest references
|
|
sha256 as an entry payload.
|
|
|
|
Backs the resource-bound token check on GET /payload: a token pinned to its
|
|
own scope(s) may only pull blobs those scopes actually ship, not any blob by
|
|
hash. Empty scopenames -> False (a bound-but-empty token reaches nothing).
|
|
"""
|
|
if not scopenames:
|
|
return False
|
|
rows = db.session.query(ManifestPublishedVersion).join(
|
|
ManifestScope,
|
|
ManifestPublishedVersion.scopeid == ManifestScope.scopeid,
|
|
).filter(
|
|
ManifestScope.scopename.in_(list(scopenames)),
|
|
ManifestPublishedVersion.iscurrent == True, # noqa: E712
|
|
).all()
|
|
for row in rows:
|
|
try:
|
|
doc = json.loads(row.manifestjson)
|
|
except (ValueError, TypeError):
|
|
continue
|
|
for entry in doc.get('Applications', []) or []:
|
|
if entry.get('PayloadSha256') == sha256:
|
|
return True
|
|
return False
|
|
|
|
|
|
def store_blob(rawbytes, filename, contenttype=None):
|
|
"""Store bytes in the content-addressed payload store; return the sha256.
|
|
|
|
Deduped by content hash - an already-present blob is not rewritten. Upserts
|
|
a ManifestBlob registry row (uncommitted). Entries reference it by
|
|
payloadsha256; the client fetches GET /api/geenforce/payload/<sha256>.
|
|
"""
|
|
sha = hashlib.sha256(rawbytes).hexdigest()
|
|
path = blob_path(sha)
|
|
if not os.path.exists(path):
|
|
fd, tmp = tempfile.mkstemp(dir=_payload_dir(), suffix='.tmp')
|
|
try:
|
|
with os.fdopen(fd, 'wb') as handle:
|
|
handle.write(rawbytes)
|
|
os.replace(tmp, path)
|
|
except Exception:
|
|
if os.path.exists(tmp):
|
|
os.remove(tmp)
|
|
raise
|
|
if not db.session.get(ManifestBlob, sha):
|
|
db.session.add(ManifestBlob(
|
|
sha256=sha, filename=filename, contenttype=contenttype,
|
|
sizebytes=len(rawbytes), createdat=_utcnow()))
|
|
return sha
|
|
|
|
|
|
def store_inline_payload(entry, filename, contenttype, rawbytes):
|
|
"""Replace the entry's inline payload with these bytes (uncommitted).
|
|
|
|
Computes payloadsha256, upserts the single ManifestPayload row (the table
|
|
has no unique constraint on entryid, so the one-inline-payload-per-entry
|
|
rule is enforced here), and points the entry at it (payloadsource='inline').
|
|
Returns the ManifestPayload.
|
|
"""
|
|
sha = hashlib.sha256(rawbytes).hexdigest()
|
|
# Flush any pending inserts (e.g. a prior entry's inline payload) BEFORE the
|
|
# bulk delete. Otherwise the delete's autoflush interleaves that half-built
|
|
# INSERT and, on MySQL, fails the manifestpayloads->manifestentries FK
|
|
# (SQLite does not enforce it the same way, so tests miss this). Then delete
|
|
# existing rows for this entry with autoflush off + no session-sync.
|
|
db.session.flush()
|
|
with db.session.no_autoflush:
|
|
ManifestPayload.query.filter_by(
|
|
entryid=entry.entryid).delete(synchronize_session=False)
|
|
db.session.flush()
|
|
payload = ManifestPayload(
|
|
entryid=entry.entryid,
|
|
filename=filename,
|
|
contenttype=contenttype,
|
|
payloadbytes=rawbytes,
|
|
payloadsha256=sha,
|
|
uploadedat=_utcnow())
|
|
db.session.add(payload)
|
|
entry.payloadsource = 'inline'
|
|
entry.payloadref = filename
|
|
entry.payloadsha256 = sha
|
|
return payload
|
|
|
|
|
|
def export_scope_to_share(scopename, phase, shareroot):
|
|
"""Write a scope's current published JSON to the share, backing up the old
|
|
file to _meta/history first. Returns the written path."""
|
|
scope = ManifestScope.query.filter_by(
|
|
scopename=scopename, phase=phase).first()
|
|
if not scope:
|
|
raise ValueError(f'No scope {scopename}/{phase}')
|
|
published = scope.publishedversions.filter_by(iscurrent=True).first()
|
|
if not published:
|
|
raise ValueError(f'{scopename} has no published version')
|
|
|
|
if phase == 'preinstall':
|
|
target = os.path.join(shareroot, 'preinstall.json')
|
|
else:
|
|
target = os.path.join(shareroot, scopename, 'manifest.json')
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
|
|
if os.path.isfile(target):
|
|
historydir = os.path.join(shareroot, '_meta', 'history')
|
|
os.makedirs(historydir, exist_ok=True)
|
|
stamp = _utcnow().strftime('%Y%m%d-%H%M%S')
|
|
with open(target) as src:
|
|
old = src.read()
|
|
with open(os.path.join(historydir, f'{stamp}-{scopename}.json'), 'w') as dst:
|
|
dst.write(old)
|
|
|
|
# Atomic write: a partial/failed write must never leave the live on-share
|
|
# manifest (which every PC reads) truncated. Write a temp file in the same
|
|
# directory, then rename over the target.
|
|
targetdir = os.path.dirname(target)
|
|
fd, tmppath = tempfile.mkstemp(dir=targetdir, suffix='.tmp')
|
|
try:
|
|
with os.fdopen(fd, 'w') as handle:
|
|
handle.write(published.manifestjson)
|
|
os.replace(tmppath, target)
|
|
except Exception:
|
|
if os.path.exists(tmppath):
|
|
os.remove(tmppath)
|
|
raise
|
|
return target
|