6c975a107c
The stale-backup card could not be built as designed, and the reason is more important than the card. Dedup means an unchanged configuration writes no revision, so collectedat moves only on a CHANGE. A machine stable for six months has a six-month-old newest revision and is perfectly healthy. Keying a staleness card on revision age would have flagged most of the fleet - exactly the noise that makes a board worth ignoring. Underneath that: ShopDB could not distinguish those cases at all. On a no-op the server returned "unchanged" and wrote nothing, so "we checked yesterday and it matched" was discarded. That fact is the one thing a backup system must be able to prove, and the only record of it was a line in a log file on the PC. lastseenat records the check rather than the change. Touched on every matching post including the no-op; set on creation, since a new revision has by definition just been seen; backfilled from collectedat or createdat so existing rows start from the last moment the config can be PROVEN current, rather than from now - claiming a check that never happened would be worse than silence. The card keys on it, one row per CHAIN rather than per asset: a machine with two part markers can have one still reporting while the other stopped, and a per-asset view would report the machine as fine. It stays deliberately silent about assets never backed up, because whether one SHOULD be is a question only the manifest can answer, and guessing would list a hundred healthy machines. The rule lives in services/staleness.py rather than the route, so it is testable without an auth layer in the way - the same split retention.py uses. Threshold is backups_staledays, default 3, and 0 disables the card.
409 lines
18 KiB
Python
409 lines
18 KiB
Python
"""Backups plugin main class.
|
|
|
|
Per-asset configuration backups with revision history, pluggable by kind
|
|
(services/registry.py). Accepts collector input over HTTPS per ADR-006 and
|
|
renders one asset panel per kind via the ADR-010 hook.
|
|
|
|
Dedup is the load-bearing behaviour: GE-Enforce runs the collector every cycle
|
|
across the fleet, so apply_collector_payload inserts a revision only when the
|
|
content hash differs from that asset's latest for that kind. Without it a
|
|
readable history would be buried under millions of identical rows.
|
|
"""
|
|
|
|
import base64
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Type
|
|
|
|
from flask import Flask, Blueprint
|
|
|
|
from shopdb.plugins.base import BasePlugin, PluginMeta
|
|
|
|
from .api import backups_bp
|
|
from .models import BackupRevision
|
|
from .services.registry import (REGISTRY, getkind, canonicalhash,
|
|
DEFAULTSHAREROOT)
|
|
from .services.retention import prune, samesource as _samesource
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BackupsPlugin(BasePlugin):
|
|
"""Configuration backup + revision history plugin."""
|
|
|
|
def __init__(self):
|
|
self._manifest = self._load_manifest()
|
|
|
|
def _load_manifest(self) -> Dict:
|
|
manifest_path = Path(__file__).parent / 'manifest.json'
|
|
if manifest_path.exists():
|
|
with open(manifest_path, 'r') as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
@property
|
|
def meta(self) -> PluginMeta:
|
|
return PluginMeta(
|
|
name=self._manifest.get('name', 'backups'),
|
|
version=self._manifest.get('version', '1.0.0'),
|
|
description=self._manifest.get('description', 'Configuration backups'),
|
|
author=self._manifest.get('author', 'ShopDB Team'),
|
|
dependencies=self._manifest.get('dependencies', []),
|
|
core_version=self._manifest.get('core_version', '>=0.2.0,<1.0.0'),
|
|
api_prefix=self._manifest.get('api_prefix', '/api/backups'),
|
|
)
|
|
|
|
def get_blueprint(self) -> Optional[Blueprint]:
|
|
return backups_bp
|
|
|
|
def get_models(self) -> List[Type]:
|
|
return [BackupRevision]
|
|
|
|
def get_permissions(self) -> List:
|
|
"""RBAC for the human-facing routes.
|
|
|
|
Deliberately split: download is separate from view because a .reg is a
|
|
restorable artifact, so a read-only auditor should be able to see that a
|
|
machine's config changed without being able to pull a file that
|
|
reconfigures it.
|
|
|
|
No delete permission. Revisions are removed only by the retention
|
|
policy; a hand-delete route would let someone quietly drop the evidence
|
|
of a config change, and there is no operational need for it. Collector
|
|
INGEST is not covered here at all - that path authorizes on a
|
|
collector.ingest-scoped service token, not a user JWT.
|
|
"""
|
|
return [
|
|
('backups.view', 'View configuration backups', 'backups'),
|
|
('backups.download', 'Download configuration backups', 'backups'),
|
|
]
|
|
|
|
def get_asset_panels(self) -> List[Dict]:
|
|
"""One panel per kind, plus the DNC Info card.
|
|
|
|
DNC Info sits ABOVE the history panels (lower position) because it
|
|
answers the question a tech actually arrives with - what is this
|
|
machine's controller talking to - while the history panels are for the
|
|
rarer restore case.
|
|
"""
|
|
panels = [kind.infopanel() for kind in REGISTRY.values()
|
|
if kind.infopanel()]
|
|
for position, kind in enumerate(REGISTRY.values()):
|
|
panel = {
|
|
'id': 'backups-{}'.format(kind.key),
|
|
'title': kind.displayname,
|
|
'assettypes': list(kind.assettypes),
|
|
'endpoint': '/api/backups/asset/{assetid}?kind=' + kind.key,
|
|
'render': 'list',
|
|
'map': {
|
|
'title': 'label',
|
|
'meta': [
|
|
{'key': 'collectedat', 'label': 'Captured', 'format': 'date'},
|
|
{'key': 'sourcehostname', 'label': 'From'},
|
|
{'key': 'shorthash', 'label': 'Hash', 'mono': True},
|
|
],
|
|
},
|
|
# The generic renderer has no per-item actions, and downloading
|
|
# a .reg needs a per-revision control plus a dialect choice, so
|
|
# the panel links to the plugin's own history view instead.
|
|
'manage': {
|
|
'to': '/backups/asset/{assetid}',
|
|
'label': 'History / download',
|
|
'emptylabel': 'View history',
|
|
},
|
|
'position': 40 + position,
|
|
}
|
|
# OMIT 'empty' entirely unless the kind sets it. The list renderer
|
|
# shows a panel when it has rows OR declares empty text, so a
|
|
# present-but-null 'empty' would still render the panel on every
|
|
# asset of the type. Most machines are not part markers and never
|
|
# will be, so that panel must disappear rather than announce it has
|
|
# nothing.
|
|
if kind.emptytext:
|
|
panel['empty'] = kind.emptytext
|
|
panels.append(panel)
|
|
return panels
|
|
|
|
def get_settings_defaults(self) -> List[Dict]:
|
|
"""Schedule + retention, read by the collector and the prune job.
|
|
|
|
The interval lives here rather than in the manifest entry because
|
|
GE-Enforce fires every cycle (several times a day on a shift change).
|
|
The collector reads backups_intervalhours and no-ops when its marker
|
|
file is younger than that, so the cadence is changed centrally in
|
|
ShopDB instead of by editing a script on the share.
|
|
"""
|
|
return [
|
|
{
|
|
'key': 'backups_intervalhours',
|
|
'value': '24',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Minimum hours between collection attempts on a '
|
|
'PC. GE-Enforce runs every cycle; the collector '
|
|
'skips until this much time has passed.',
|
|
# MUST be public. The collecting script reads this from
|
|
# /api/settings/public BEFORE it has any credential, and its
|
|
# Get-IntervalHours falls back to 24 on any failure - silently.
|
|
# Left off the allowlist, the setting looked configurable and
|
|
# was not: every PC used 24 no matter what the UI said. A
|
|
# collection cadence is not a secret; the share root beneath it
|
|
# is internal topology and stays private.
|
|
'public': True,
|
|
},
|
|
{
|
|
'key': 'backups_shareroot',
|
|
'value': DEFAULTSHAREROOT,
|
|
'valuetype': 'string',
|
|
'category': 'backups',
|
|
'description': 'UNC root that opaque (non-JSON) backups are '
|
|
'written under by the collecting PC. Site-specific.',
|
|
},
|
|
{
|
|
'key': 'backups_staledays',
|
|
'value': '3',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Days without a CONFIRMED check before a backup '
|
|
'is listed as stopped on the dashboard. Measured '
|
|
'from the last check, not the last change - an '
|
|
'unchanged config writes no revision. 0 disables '
|
|
'the card.',
|
|
},
|
|
{
|
|
'key': 'backups_retentioncount',
|
|
'value': '50',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Distinct revisions kept per asset per kind. '
|
|
'0 keeps everything. The newest and the oldest '
|
|
'are never pruned.',
|
|
},
|
|
{
|
|
'key': 'backups_retentiondays',
|
|
'value': '0',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Also prune revisions older than this many days. '
|
|
'0 disables age-based pruning.',
|
|
},
|
|
]
|
|
|
|
def get_dashboard_widgets(self) -> List[Dict]:
|
|
"""Dashboard card: backups that have stopped running.
|
|
|
|
Keyed on the last CONFIRMED check, never on the last change. Dedup means
|
|
an unchanged config writes no revision, so a card keyed on revision age
|
|
would flag most of a healthy fleet.
|
|
"""
|
|
return [
|
|
{
|
|
'id': 'backups-stale',
|
|
'title': 'Backups stopped',
|
|
'endpoint': '/api/backups/dashboard/stale',
|
|
'render': 'exceptions',
|
|
'severity': 'warning',
|
|
'permission': 'backups.view',
|
|
'empty': 'hide',
|
|
'position': 30,
|
|
'map': {
|
|
'title': 'assetnumber',
|
|
'detail': 'backupkind',
|
|
'meta': [
|
|
{'key': 'sourcehostname', 'label': 'from'},
|
|
{'key': 'quietdays', 'label': 'last checked',
|
|
'suffix': ' days ago'},
|
|
],
|
|
'link': '/backups/asset/{assetid}',
|
|
},
|
|
},
|
|
]
|
|
|
|
# ---- ADR-006 collector contract -------------------------------------
|
|
|
|
def get_collector_schema(self) -> Optional[dict]:
|
|
return {
|
|
'identityfield': 'machinenumber',
|
|
'fields': {
|
|
'type': 'object',
|
|
'required': ['machinenumber', 'backupkind'],
|
|
'properties': {
|
|
'machinenumber': {
|
|
'type': 'string',
|
|
'description': 'Machine number the config belongs to. '
|
|
'Resolved against assets.assetnumber.',
|
|
},
|
|
'backupkind': {
|
|
'type': 'string',
|
|
'enum': sorted(REGISTRY.keys()),
|
|
},
|
|
'contentbase64': {
|
|
'type': 'string',
|
|
'description': "Raw backup file, base64. Required for "
|
|
"'shopdb' kinds (parsed server-side).",
|
|
},
|
|
'contenthash': {
|
|
'type': 'string',
|
|
'description': "sha256 of the file. Required for 'share' "
|
|
"kinds, whose bytes stay on the SFLD share.",
|
|
},
|
|
'sharepath': {
|
|
'type': 'string',
|
|
'description': "Full UNC path. Required for 'share' kinds.",
|
|
},
|
|
'sourcefilename': {'type': 'string'},
|
|
'sourcehostname': {
|
|
'type': 'string',
|
|
'description': 'PC the backup was read from. For NTLARS '
|
|
'this is the controlling PC, not the machine.',
|
|
},
|
|
'collectedat': {'type': 'string', 'format': 'date-time'},
|
|
'bytesize': {'type': 'integer'},
|
|
},
|
|
},
|
|
}
|
|
|
|
def apply_collector_payload(self, payload: dict) -> dict:
|
|
from shopdb.api import db
|
|
|
|
warnings = []
|
|
kindkey = (payload.get('backupkind') or '').strip().lower()
|
|
kind = getkind(kindkey)
|
|
if kind is None:
|
|
raise ValueError('unknown backupkind: {!r}'.format(kindkey))
|
|
|
|
assetid, err = kind.resolveassetid(payload)
|
|
if assetid is None:
|
|
# Unresolvable is reported, never silently dropped - an unmatched
|
|
# machine number means a real gap between the fleet and ShopDB.
|
|
raise ValueError('could not resolve asset: {}'.format(err))
|
|
|
|
projection = None
|
|
sharepath = None
|
|
sourcefilename = payload.get('sourcefilename')
|
|
bytesize = payload.get('bytesize')
|
|
|
|
if kind.storagebackend == 'shopdb':
|
|
encoded = payload.get('contentbase64')
|
|
if not encoded:
|
|
raise ValueError(
|
|
"contentbase64 is required for kind '{}'".format(kindkey))
|
|
raw = base64.b64decode(encoded)
|
|
bytesize = bytesize or len(raw)
|
|
projection = kind.parse(raw)
|
|
contenthash = canonicalhash(projection)
|
|
|
|
# The fleet's machine number (pc-config.txt) is authoritative for
|
|
# WHICH asset this belongs to, but NTLARS carries its own MachineNo.
|
|
# A disagreement means the PC is running another machine's config -
|
|
# worth surfacing rather than quietly filing under the reported one.
|
|
embedded = getattr(kind, 'embeddedmachineno', None)
|
|
if embedded is not None:
|
|
embeddedno = embedded(projection)
|
|
reported = (payload.get('machinenumber') or '').strip()
|
|
if embeddedno and reported and embeddedno != reported:
|
|
warnings.append(
|
|
'reported machine number {!r} but NTLARS is configured '
|
|
'for {!r}'.format(reported, embeddedno))
|
|
else:
|
|
contenthash = (payload.get('contenthash') or '').strip().lower()
|
|
sharepath = (payload.get('sharepath') or '').strip()
|
|
if not contenthash or not sharepath:
|
|
raise ValueError(
|
|
"contenthash and sharepath are required for kind '{}'".format(
|
|
kindkey))
|
|
|
|
# Dedup is against the LATEST row of THIS SOURCE's chain, not the
|
|
# asset's. Several PCs legitimately share one machine number here - the
|
|
# part markers on 0613, 0615 and WJPRT do, and their configs genuinely
|
|
# differ, typically by COM port. Keyed on
|
|
# asset alone, each marker's post differed from whichever marker posted
|
|
# last, so nothing ever deduped and the chain grew by one revision per
|
|
# PC per collection cycle. Keyed on the source, an unchanged config is a
|
|
# no-op again and each marker keeps its own history against the machine.
|
|
#
|
|
# Dedup is against the latest row only, not any row, because a config
|
|
# reverting to an earlier state is a legitimate new revision.
|
|
#
|
|
# with_for_update serializes concurrent posts for the same chain: the
|
|
# fleet collects on one GE-Enforce cycle, so a retry could otherwise
|
|
# pass the hash check twice and insert the same revision.
|
|
sourcehostname = payload.get('sourcehostname')
|
|
latest = (db.session.query(BackupRevision)
|
|
.filter(BackupRevision.assetid == assetid,
|
|
BackupRevision.backupkind == kindkey,
|
|
_samesource(sourcehostname))
|
|
.order_by(BackupRevision.backuprevisionid.desc())
|
|
.with_for_update()
|
|
.first())
|
|
|
|
if latest is not None and latest.contenthash == contenthash:
|
|
# Unchanged, so no revision - but RECORD THE CHECK. Without this the
|
|
# only evidence a backup still runs is a line in a log on the PC,
|
|
# and a config stable for six months is indistinguishable from a
|
|
# backup that died six months ago. Cheap: one column on a row that
|
|
# already exists, no new history.
|
|
latest.lastseenat = datetime.utcnow()
|
|
db.session.flush()
|
|
return {
|
|
'action': 'noop',
|
|
'assetid': assetid,
|
|
'backuprevisionid': latest.backuprevisionid,
|
|
'warnings': warnings,
|
|
}
|
|
|
|
collectedat = None
|
|
rawcollected = payload.get('collectedat')
|
|
if rawcollected:
|
|
try:
|
|
collectedat = datetime.fromisoformat(rawcollected.replace('Z', '+00:00'))
|
|
if collectedat.tzinfo is not None:
|
|
# Convert, don't just drop the offset: createdat is utcnow(),
|
|
# so keeping local wall time would skew a -04:00 bay by 4h
|
|
# against its own history.
|
|
collectedat = collectedat.astimezone(timezone.utc).replace(
|
|
tzinfo=None)
|
|
except ValueError:
|
|
warnings.append('unparseable collectedat: {!r}'.format(rawcollected))
|
|
|
|
revision = BackupRevision(
|
|
assetid=assetid,
|
|
backupkind=kindkey,
|
|
storagebackend=kind.storagebackend,
|
|
contenthash=contenthash,
|
|
sharepath=sharepath,
|
|
sourcefilename=sourcefilename,
|
|
bytesize=bytesize,
|
|
sourcehostname=payload.get('sourcehostname'),
|
|
collectedat=collectedat or datetime.utcnow(),
|
|
lastseenat=datetime.utcnow(),
|
|
)
|
|
revision.payload = projection
|
|
db.session.add(revision)
|
|
# flush, not commit: the collector dispatcher owns the transaction and
|
|
# commits after writing its AuditLog row. Committing here would leave an
|
|
# unaudited revision behind if that write then failed.
|
|
db.session.flush()
|
|
|
|
pruned = prune(
|
|
assetid, kindkey,
|
|
retentioncount=self.get_setting('backups_retentioncount', 0),
|
|
retentiondays=self.get_setting('backups_retentiondays', 0),
|
|
)
|
|
if pruned:
|
|
warnings.append('pruned {} old revision(s) per retention policy'
|
|
.format(pruned))
|
|
|
|
return {
|
|
'action': 'created',
|
|
'assetid': assetid,
|
|
'backuprevisionid': revision.backuprevisionid,
|
|
'warnings': warnings,
|
|
}
|
|
|
|
def init_app(self, app: Flask, db_instance) -> None:
|
|
logger.info('Backups plugin initialized (v%s, kinds: %s)',
|
|
self.meta.version, ', '.join(sorted(REGISTRY)))
|