fca775c737
Adds a kind-pluggable backups plugin. Configuration captured from a PC is filed against the MACHINE it controls, with a revision history and download back to the native format. NTLARS/DNC is the first kind. Settings live in the controlling PC's registry but describe the machine, so revisions attach to the machine's asset and carry no foreign key to the PC: history survives a PC being replaced or deleted, and sourcehostname records the handover. Storage splits by kind. Parseable kinds store a dialect-neutral JSON projection in ShopDB and re-render on download; opaque vendor formats (part marker and similar) keep their bytes on the SFLD share with ShopDB holding metadata and the UNC pointer. Two .reg dialects exist in the wild: NTLARS's own Save... export omits the WOW6432Node path segment, scripted exports include it. Parsing strips whichever root matched, so a stored revision commits to neither and download offers both (NTLARS Load... by default, WOW6432Node for direct reg import). Getting this backwards is silent, so the dedup hash deliberately excludes sourcedialect and both dialects of one config dedup to a single revision. Dedup is load-bearing: the collector runs every GE-Enforce cycle across the fleet, so a revision is inserted only when the content hash differs from that asset's latest for that kind. A freshly imaged PC opens NTLARS with a blank General tab. Recording that would make an empty config the newest revision exactly when someone needs the last good one, so a blank MachineNo is rejected rather than accepted as a change. Two of the 320 known-good backups on the share already have that shape. DNC Info card summarises the latest revision on the machine page: General (Cnc, NcIF, HostType), eFocas, Serial, NTSHR when populated (only 18 of 147 machines), and MARK when the machine is a marker. MARK is gated on Cnc=MARKER or the ShopDB machine type, not on the MARK key having content: MARK carries serial defaults on 145 of 147 machines and DncPatterns reads YES on 103 including ordinary lathes, so neither identifies a marker. The info card is owned by the kind (BackupKind.infopanel/buildinfo) and served by a generic endpoint, so the expected successor to DNC ships its own card by adding a class rather than changing the plugin or the panel wiring. Also: schedule and retention settings with a prune that never drops the newest or the oldest revision, and scripts/import_ntlars_backups.py to seed history from the existing per-machine .reg files (144 of 147 resolve to assets). Codec verified against all 320 real backups: round-trips clean through both dialects. Bay-side generation verified on Windows against reg.exe export.
333 lines
14 KiB
Python
333 lines
14 KiB
Python
"""Backups plugin main class.
|
|
|
|
Per-asset configuration backups with revision history, pluggable by kind
|
|
(services/registry.py). Accepts collector input over HTTPS per ADR-006 and
|
|
renders one asset panel per kind via the ADR-010 hook.
|
|
|
|
Dedup is the load-bearing behaviour: GE-Enforce runs the collector every cycle
|
|
across the fleet, so apply_collector_payload inserts a revision only when the
|
|
content hash differs from that asset's latest for that kind. Without it a
|
|
readable history would be buried under millions of identical rows.
|
|
"""
|
|
|
|
import base64
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Type
|
|
|
|
from flask import Flask, Blueprint
|
|
|
|
from shopdb.plugins.base import BasePlugin, PluginMeta
|
|
|
|
from .api import backups_bp
|
|
from .models import BackupRevision
|
|
from .services.registry import (REGISTRY, getkind, canonicalhash,
|
|
DEFAULTSHAREROOT)
|
|
from .services.retention import prune
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BackupsPlugin(BasePlugin):
|
|
"""Configuration backup + revision history plugin."""
|
|
|
|
def __init__(self):
|
|
self._manifest = self._load_manifest()
|
|
|
|
def _load_manifest(self) -> Dict:
|
|
manifest_path = Path(__file__).parent / 'manifest.json'
|
|
if manifest_path.exists():
|
|
with open(manifest_path, 'r') as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
@property
|
|
def meta(self) -> PluginMeta:
|
|
return PluginMeta(
|
|
name=self._manifest.get('name', 'backups'),
|
|
version=self._manifest.get('version', '1.0.0'),
|
|
description=self._manifest.get('description', 'Configuration backups'),
|
|
author=self._manifest.get('author', 'ShopDB Team'),
|
|
dependencies=self._manifest.get('dependencies', []),
|
|
core_version=self._manifest.get('core_version', '>=0.2.0,<1.0.0'),
|
|
api_prefix=self._manifest.get('api_prefix', '/api/backups'),
|
|
)
|
|
|
|
def get_blueprint(self) -> Optional[Blueprint]:
|
|
return backups_bp
|
|
|
|
def get_models(self) -> List[Type]:
|
|
return [BackupRevision]
|
|
|
|
def get_permissions(self) -> List:
|
|
"""RBAC for the human-facing routes.
|
|
|
|
Deliberately split: download is separate from view because a .reg is a
|
|
restorable artifact, so a read-only auditor should be able to see that a
|
|
machine's config changed without being able to pull a file that
|
|
reconfigures it.
|
|
|
|
No delete permission. Revisions are removed only by the retention
|
|
policy; a hand-delete route would let someone quietly drop the evidence
|
|
of a config change, and there is no operational need for it. Collector
|
|
INGEST is not covered here at all - that path authorizes on a
|
|
collector.ingest-scoped service token, not a user JWT.
|
|
"""
|
|
return [
|
|
('backups.view', 'View configuration backups', 'backups'),
|
|
('backups.download', 'Download configuration backups', 'backups'),
|
|
]
|
|
|
|
def get_asset_panels(self) -> List[Dict]:
|
|
"""One panel per kind, plus the DNC Info card.
|
|
|
|
DNC Info sits ABOVE the history panels (lower position) because it
|
|
answers the question a tech actually arrives with - what is this
|
|
machine's controller talking to - while the history panels are for the
|
|
rarer restore case.
|
|
"""
|
|
panels = [kind.infopanel() for kind in REGISTRY.values()
|
|
if kind.infopanel()]
|
|
for position, kind in enumerate(REGISTRY.values()):
|
|
panels.append({
|
|
'id': 'backups-{}'.format(kind.key),
|
|
'title': kind.displayname,
|
|
'assettypes': list(kind.assettypes),
|
|
'endpoint': '/api/backups/asset/{assetid}?kind=' + kind.key,
|
|
'render': 'list',
|
|
'map': {
|
|
'title': 'label',
|
|
'meta': [
|
|
{'key': 'collectedat', 'label': 'Captured', 'format': 'date'},
|
|
{'key': 'sourcehostname', 'label': 'From'},
|
|
{'key': 'shorthash', 'label': 'Hash', 'mono': True},
|
|
],
|
|
},
|
|
'empty': kind.emptytext,
|
|
# The generic renderer has no per-item actions, and downloading
|
|
# a .reg needs a per-revision control plus a dialect choice, so
|
|
# the panel links to the plugin's own history view instead.
|
|
'manage': {
|
|
'to': '/backups/asset/{assetid}',
|
|
'label': 'History / download',
|
|
'emptylabel': 'View history',
|
|
},
|
|
'position': 40 + position,
|
|
})
|
|
return panels
|
|
|
|
def get_settings_defaults(self) -> List[Dict]:
|
|
"""Schedule + retention, read by the collector and the prune job.
|
|
|
|
The interval lives here rather than in the manifest entry because
|
|
GE-Enforce fires every cycle (several times a day on a shift change).
|
|
The collector reads backups_intervalhours and no-ops when its marker
|
|
file is younger than that, so the cadence is changed centrally in
|
|
ShopDB instead of by editing a script on the share.
|
|
"""
|
|
return [
|
|
{
|
|
'key': 'backups_intervalhours',
|
|
'value': '24',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Minimum hours between collection attempts on a '
|
|
'PC. GE-Enforce runs every cycle; the collector '
|
|
'skips until this much time has passed.',
|
|
},
|
|
{
|
|
'key': 'backups_shareroot',
|
|
'value': DEFAULTSHAREROOT,
|
|
'valuetype': 'string',
|
|
'category': 'backups',
|
|
'description': 'UNC root that opaque (non-JSON) backups are '
|
|
'written under by the collecting PC. Site-specific.',
|
|
},
|
|
{
|
|
'key': 'backups_retentioncount',
|
|
'value': '50',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Distinct revisions kept per asset per kind. '
|
|
'0 keeps everything. The newest and the oldest '
|
|
'are never pruned.',
|
|
},
|
|
{
|
|
'key': 'backups_retentiondays',
|
|
'value': '0',
|
|
'valuetype': 'integer',
|
|
'category': 'backups',
|
|
'description': 'Also prune revisions older than this many days. '
|
|
'0 disables age-based pruning.',
|
|
},
|
|
]
|
|
|
|
# ---- ADR-006 collector contract -------------------------------------
|
|
|
|
def get_collector_schema(self) -> Optional[dict]:
|
|
return {
|
|
'identityfield': 'machinenumber',
|
|
'fields': {
|
|
'type': 'object',
|
|
'required': ['machinenumber', 'backupkind'],
|
|
'properties': {
|
|
'machinenumber': {
|
|
'type': 'string',
|
|
'description': 'Machine number the config belongs to. '
|
|
'Resolved against assets.assetnumber.',
|
|
},
|
|
'backupkind': {
|
|
'type': 'string',
|
|
'enum': sorted(REGISTRY.keys()),
|
|
},
|
|
'contentbase64': {
|
|
'type': 'string',
|
|
'description': "Raw backup file, base64. Required for "
|
|
"'shopdb' kinds (parsed server-side).",
|
|
},
|
|
'contenthash': {
|
|
'type': 'string',
|
|
'description': "sha256 of the file. Required for 'share' "
|
|
"kinds, whose bytes stay on the SFLD share.",
|
|
},
|
|
'sharepath': {
|
|
'type': 'string',
|
|
'description': "Full UNC path. Required for 'share' kinds.",
|
|
},
|
|
'sourcefilename': {'type': 'string'},
|
|
'sourcehostname': {
|
|
'type': 'string',
|
|
'description': 'PC the backup was read from. For NTLARS '
|
|
'this is the controlling PC, not the machine.',
|
|
},
|
|
'collectedat': {'type': 'string', 'format': 'date-time'},
|
|
'bytesize': {'type': 'integer'},
|
|
},
|
|
},
|
|
}
|
|
|
|
def apply_collector_payload(self, payload: dict) -> dict:
|
|
from shopdb.api import db
|
|
|
|
warnings = []
|
|
kindkey = (payload.get('backupkind') or '').strip().lower()
|
|
kind = getkind(kindkey)
|
|
if kind is None:
|
|
raise ValueError('unknown backupkind: {!r}'.format(kindkey))
|
|
|
|
assetid, err = kind.resolveassetid(payload)
|
|
if assetid is None:
|
|
# Unresolvable is reported, never silently dropped - an unmatched
|
|
# machine number means a real gap between the fleet and ShopDB.
|
|
raise ValueError('could not resolve asset: {}'.format(err))
|
|
|
|
projection = None
|
|
sharepath = None
|
|
sourcefilename = payload.get('sourcefilename')
|
|
bytesize = payload.get('bytesize')
|
|
|
|
if kind.storagebackend == 'shopdb':
|
|
encoded = payload.get('contentbase64')
|
|
if not encoded:
|
|
raise ValueError(
|
|
"contentbase64 is required for kind '{}'".format(kindkey))
|
|
raw = base64.b64decode(encoded)
|
|
bytesize = bytesize or len(raw)
|
|
projection = kind.parse(raw)
|
|
contenthash = canonicalhash(projection)
|
|
|
|
# The fleet's machine number (pc-config.txt) is authoritative for
|
|
# WHICH asset this belongs to, but NTLARS carries its own MachineNo.
|
|
# A disagreement means the PC is running another machine's config -
|
|
# worth surfacing rather than quietly filing under the reported one.
|
|
embedded = getattr(kind, 'embeddedmachineno', None)
|
|
if embedded is not None:
|
|
embeddedno = embedded(projection)
|
|
reported = (payload.get('machinenumber') or '').strip()
|
|
if embeddedno and reported and embeddedno != reported:
|
|
warnings.append(
|
|
'reported machine number {!r} but NTLARS is configured '
|
|
'for {!r}'.format(reported, embeddedno))
|
|
else:
|
|
contenthash = (payload.get('contenthash') or '').strip().lower()
|
|
sharepath = (payload.get('sharepath') or '').strip()
|
|
if not contenthash or not sharepath:
|
|
raise ValueError(
|
|
"contenthash and sharepath are required for kind '{}'".format(
|
|
kindkey))
|
|
|
|
# with_for_update serializes concurrent posts for the same asset+kind.
|
|
# The whole fleet collects on the same GE-Enforce cycle, so a retry or
|
|
# two PCs claiming one machine number can otherwise both pass the hash
|
|
# check and insert duplicate identical revisions. No unique constraint
|
|
# can cover this: dedup is against the LATEST row only, because a config
|
|
# reverting to an earlier state is a legitimate new revision.
|
|
latest = (db.session.query(BackupRevision)
|
|
.filter(BackupRevision.assetid == assetid,
|
|
BackupRevision.backupkind == kindkey)
|
|
.order_by(BackupRevision.backuprevisionid.desc())
|
|
.with_for_update()
|
|
.first())
|
|
|
|
if latest is not None and latest.contenthash == contenthash:
|
|
return {
|
|
'action': 'noop',
|
|
'assetid': assetid,
|
|
'backuprevisionid': latest.backuprevisionid,
|
|
'warnings': warnings,
|
|
}
|
|
|
|
collectedat = None
|
|
rawcollected = payload.get('collectedat')
|
|
if rawcollected:
|
|
try:
|
|
collectedat = datetime.fromisoformat(rawcollected.replace('Z', '+00:00'))
|
|
if collectedat.tzinfo is not None:
|
|
# Convert, don't just drop the offset: createdat is utcnow(),
|
|
# so keeping local wall time would skew a -04:00 bay by 4h
|
|
# against its own history.
|
|
collectedat = collectedat.astimezone(timezone.utc).replace(
|
|
tzinfo=None)
|
|
except ValueError:
|
|
warnings.append('unparseable collectedat: {!r}'.format(rawcollected))
|
|
|
|
revision = BackupRevision(
|
|
assetid=assetid,
|
|
backupkind=kindkey,
|
|
storagebackend=kind.storagebackend,
|
|
contenthash=contenthash,
|
|
sharepath=sharepath,
|
|
sourcefilename=sourcefilename,
|
|
bytesize=bytesize,
|
|
sourcehostname=payload.get('sourcehostname'),
|
|
collectedat=collectedat or datetime.utcnow(),
|
|
)
|
|
revision.payload = projection
|
|
db.session.add(revision)
|
|
# flush, not commit: the collector dispatcher owns the transaction and
|
|
# commits after writing its AuditLog row. Committing here would leave an
|
|
# unaudited revision behind if that write then failed.
|
|
db.session.flush()
|
|
|
|
pruned = prune(
|
|
assetid, kindkey,
|
|
retentioncount=self.get_setting('backups_retentioncount', 0),
|
|
retentiondays=self.get_setting('backups_retentiondays', 0),
|
|
)
|
|
if pruned:
|
|
warnings.append('pruned {} old revision(s) per retention policy'
|
|
.format(pruned))
|
|
|
|
return {
|
|
'action': 'created',
|
|
'assetid': assetid,
|
|
'backuprevisionid': revision.backuprevisionid,
|
|
'warnings': warnings,
|
|
}
|
|
|
|
def init_app(self, app: Flask, db_instance) -> None:
|
|
logger.info('Backups plugin initialized (v%s, kinds: %s)',
|
|
self.meta.version, ', '.join(sorted(REGISTRY)))
|