diff --git a/plugins/backups/__init__.py b/plugins/backups/__init__.py new file mode 100644 index 0000000..a5af74d --- /dev/null +++ b/plugins/backups/__init__.py @@ -0,0 +1,5 @@ +"""Backups plugin package.""" + +from .plugin import BackupsPlugin + +__all__ = ['BackupsPlugin'] diff --git a/plugins/backups/api/__init__.py b/plugins/backups/api/__init__.py new file mode 100644 index 0000000..f2d6c9d --- /dev/null +++ b/plugins/backups/api/__init__.py @@ -0,0 +1,5 @@ +"""Backups plugin API.""" + +from .routes import backups_bp + +__all__ = ['backups_bp'] diff --git a/plugins/backups/api/routes.py b/plugins/backups/api/routes.py new file mode 100644 index 0000000..deee70b --- /dev/null +++ b/plugins/backups/api/routes.py @@ -0,0 +1,306 @@ +"""Backups API. + +Read + download only. Revisions are created exclusively through the ADR-006 +collector endpoint (POST /api/collector/backups), so there is deliberately no +create route here: a hand-posted "backup" that never came off a machine would +poison the history this feature exists to provide. + +Downloads differ by storage backend. 'shopdb' kinds re-render from the stored +projection, which is what lets one revision produce either .reg dialect. 'share' +kinds are not streamed - ShopDB returns the UNC path for the tech to open, +because requiring the app server to mount the SFLD share would turn a +permissions slip into an unexplained empty download. +""" + +from flask import Blueprint, current_app, request, Response +from flask_jwt_extended import jwt_required + +from shopdb.api import ( + db, Asset, + success_response, error_response, ErrorCodes, + require_permission, +) + +from ..models import BackupRevision +from ..services.registry import REGISTRY, getkind + +backups_bp = Blueprint('backups', __name__) + + +def _label(revision): + """Panel list title: kind-agnostic, readable at a glance.""" + when = revision.collectedat or revision.createdat + stamp = when.strftime('%Y-%m-%d %H:%M') if when else 'unknown time' + if revision.sourcefilename: + return '{} - {}'.format(stamp, revision.sourcefilename) + return stamp + + +def _summary(revision, islatest=False): + data = revision.to_dict() + data['label'] = _label(revision) + data['islatest'] = islatest + kind = getkind(revision.backupkind) + data['formats'] = kind.formats() if kind else [] + return data + + +@backups_bp.route('/kinds', methods=['GET']) +@jwt_required() +@require_permission('backups.view') +def list_kinds(): + """Registered backup kinds, for UI that needs to enumerate them.""" + return success_response([ + { + 'key': kind.key, + 'displayname': kind.displayname, + 'storagebackend': kind.storagebackend, + 'assettypes': list(kind.assettypes), + 'formats': kind.formats(), + } + for kind in REGISTRY.values() + ]) + + +@backups_bp.route('/asset/', methods=['GET']) +@jwt_required() +@require_permission('backups.view') +def asset_revisions(assetid): + """Revision history for one asset, newest first. + + This is the asset-panel endpoint. Optional ?kind= narrows to a single kind, + which is how each per-kind panel scopes itself. + """ + asset = db.session.get(Asset, assetid) + if asset is None: + return error_response(ErrorCodes.NOT_FOUND, 'Asset not found', http_code=404) + + query = db.session.query(BackupRevision).filter( + BackupRevision.assetid == assetid) + + kindkey = (request.args.get('kind') or '').strip().lower() + if kindkey: + if getkind(kindkey) is None: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Unknown kind: {}'.format(kindkey)) + query = query.filter(BackupRevision.backupkind == kindkey) + + revisions = query.order_by(BackupRevision.backuprevisionid.desc()).all() + + # "Latest" is per kind, not per asset, so an unfiltered listing still marks + # the current revision of each kind correctly. + seen = set() + out = [] + for revision in revisions: + islatest = revision.backupkind not in seen + seen.add(revision.backupkind) + out.append(_summary(revision, islatest=islatest)) + return success_response(out) + + +@backups_bp.route('/asset//info', methods=['GET']) +@jwt_required() +@require_permission('backups.view') +def asset_info(assetid): + """A kind's 'at a glance' card, built from its LATEST revision. + + Generic across kinds: the requested kind owns both the panel declaration + and the payload (see BackupKind.infopanel / buildinfo), so a successor to + NTLARS/DNC gets its own card without a new endpoint. + + Empty when the machine has no revision of that kind yet, which the generic + renderer turns into the panel's empty text. + """ + kindkey = (request.args.get('kind') or 'ntlars').strip().lower() + kind = getkind(kindkey) + if kind is None: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Unknown kind: {}'.format(kindkey)) + + revision = (db.session.query(BackupRevision) + .filter(BackupRevision.assetid == assetid, + BackupRevision.backupkind == kindkey) + .order_by(BackupRevision.backuprevisionid.desc()) + .first()) + if revision is None: + return success_response({'fields': [], 'sectioncount': 0}) + + data = kind.buildinfo( + revision.payload, assetid, + partmarkertypes=current_app.config.get('BACKUPS_PARTMARKER_TYPES')) + data['backuprevisionid'] = revision.backuprevisionid + data['collectedat'] = (revision.collectedat.isoformat() + if revision.collectedat else None) + return success_response(data) + + +@backups_bp.route('/revisions/', methods=['GET']) +@jwt_required() +@require_permission('backups.view') +def get_revision(backuprevisionid): + """One revision, including its parsed payload where there is one.""" + revision = db.session.get(BackupRevision, backuprevisionid) + if revision is None: + return error_response(ErrorCodes.NOT_FOUND, 'Revision not found', + http_code=404) + data = revision.to_dict(includepayload=True) + data['label'] = _label(revision) + kind = getkind(revision.backupkind) + data['formats'] = kind.formats() if kind else [] + data['displayname'] = kind.displayname if kind else revision.backupkind + return success_response(data) + + +@backups_bp.route('/revisions//download', methods=['GET']) +@jwt_required() +@require_permission('backups.download') +def download_revision(backuprevisionid): + """Render a revision back to its native file format. + + ?format=ntlars (default for NTLARS) omits WOW6432Node - the form the + NTLARS Load... button expects + ?format=wow6432node includes WOW6432Node - for `reg import` on 64-bit + """ + revision = db.session.get(BackupRevision, backuprevisionid) + if revision is None: + return error_response(ErrorCodes.NOT_FOUND, 'Revision not found', + http_code=404) + + kind = getkind(revision.backupkind) + if kind is None: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Unknown kind: {}'.format(revision.backupkind)) + + if revision.storagebackend == 'share': + # Not an error: the file exists, ShopDB just is not the one serving it. + return success_response({ + 'storagebackend': 'share', + 'sharepath': revision.sharepath, + 'sourcefilename': revision.sourcefilename, + 'message': 'This backup lives on the SFLD share. Open the path directly.', + }) + + formats = kind.formats() + formatid = (request.args.get('format') or '').strip().lower() + if not formatid: + formatid = formats[0]['id'] if formats else '' + if not any(f['id'] == formatid for f in formats): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'Unknown format {!r} for kind {}'.format(formatid, kind.key)) + + asset = db.session.get(Asset, revision.assetid) + assetnumber = asset.assetnumber if asset else str(revision.assetid) + when = revision.collectedat or revision.createdat + + comments = [ + '{} backup from ShopDB'.format(kind.displayname), + 'Asset: {}'.format(assetnumber), + 'Captured: {}'.format(when.strftime('%Y-%m-%d %H:%M:%S') if when else 'unknown'), + 'Source PC: {}'.format(revision.sourcehostname or 'unknown'), + 'Revision: {} ({})'.format(revision.backuprevisionid, + (revision.contenthash or '')[:12]), + ] + + try: + raw, ext, mimetype = kind.render(revision.payload, formatid, + comments=comments) + except ValueError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + + filename = revision.sourcefilename or '{}-{}{}'.format( + assetnumber, kind.key, ext) + if formatid == 'wow6432node': + filename = '{}-{}-wow6432node{}'.format(assetnumber, kind.key, ext) + + return Response( + raw, + mimetype=mimetype, + headers={'Content-Disposition': 'attachment; filename="{}"'.format(filename)}, + ) + + +@backups_bp.route('/revisions//diff', methods=['GET']) +@jwt_required() +@require_permission('backups.view') +def diff_revision(backuprevisionid): + """Diff two revisions of the same asset+kind. + + ?against= picks the comparison revision; default is the immediately + preceding one, which answers "what changed" without the user choosing. + """ + revision = db.session.get(BackupRevision, backuprevisionid) + if revision is None: + return error_response(ErrorCodes.NOT_FOUND, 'Revision not found', + http_code=404) + if revision.storagebackend != 'shopdb': + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'Kind {} stores opaque files and cannot be diffed'.format( + revision.backupkind)) + + againstid = request.args.get('against', type=int) + if againstid: + other = db.session.get(BackupRevision, againstid) + else: + other = (db.session.query(BackupRevision) + .filter(BackupRevision.assetid == revision.assetid, + BackupRevision.backupkind == revision.backupkind, + BackupRevision.backuprevisionid < revision.backuprevisionid) + .order_by(BackupRevision.backuprevisionid.desc()) + .first()) + + if other is None: + return success_response({ + 'backuprevisionid': revision.backuprevisionid, + 'againstid': None, + 'changes': [], + 'message': 'No earlier revision to compare against.', + }) + + changes = _diffprojections(other.payload, revision.payload) + return success_response({ + 'backuprevisionid': revision.backuprevisionid, + 'againstid': other.backuprevisionid, + 'changes': changes, + 'changecount': len(changes), + }) + + +def _flatten(projection): + """{(subkey, valuename): (type, data)} from a stored projection.""" + flat = {} + for key in (projection or {}).get('keys', []): + path = key.get('path', '') + for name, entry in (key.get('values') or {}).items(): + flat[(path, name)] = (entry.get('type'), entry.get('data')) + return flat + + +def _diffprojections(old, new): + oldflat = _flatten(old) + newflat = _flatten(new) + + changes = [] + for ref in sorted(set(oldflat) | set(newflat)): + path, name = ref + before = oldflat.get(ref) + after = newflat.get(ref) + if before == after: + continue + if before is None: + change = 'added' + elif after is None: + change = 'removed' + else: + change = 'changed' + changes.append({ + 'keypath': path, + 'valuename': name, + 'change': change, + 'before': None if before is None else before[1], + 'after': None if after is None else after[1], + 'beforetype': None if before is None else before[0], + 'aftertype': None if after is None else after[0], + }) + return changes diff --git a/plugins/backups/frontend/routes.js b/plugins/backups/frontend/routes.js new file mode 100644 index 0000000..8449527 --- /dev/null +++ b/plugins/backups/frontend/routes.js @@ -0,0 +1,21 @@ +/** + * Backups plugin routes. + * + * One route: the per-asset revision history. The asset-detail panel (ADR-010 + * Path A) lists recent revisions but the generic renderer has no per-item + * actions, and downloading a .reg needs a per-revision control plus a dialect + * choice. So the panel links here via its manage link rather than the renderer + * growing a backups-shaped feature. + * + * meta.plugin = 'backups' so the ADR-009 guard redirects when the backend + * plugin is disabled. requiresAuth because every backups API route is behind + * JWT + a backups.* permission; an anonymous visit would only render errors. + */ +export default [ + { + path: 'backups/asset/:assetid', + name: 'backups-asset-history', + component: () => import('./views/BackupHistory.vue'), + meta: { requiresAuth: true, plugin: 'backups' } + } +] diff --git a/plugins/backups/frontend/views/BackupHistory.vue b/plugins/backups/frontend/views/BackupHistory.vue new file mode 100644 index 0000000..5c1a0f8 --- /dev/null +++ b/plugins/backups/frontend/views/BackupHistory.vue @@ -0,0 +1,210 @@ + + + + + diff --git a/plugins/backups/manifest.json b/plugins/backups/manifest.json new file mode 100644 index 0000000..55f2806 --- /dev/null +++ b/plugins/backups/manifest.json @@ -0,0 +1,14 @@ +{ + "name": "backups", + "version": "1.0.0", + "description": "Per-asset configuration backups with revision history. Kind-pluggable: NTLARS/DNC registry, part marker, UDC, CMM. Parseable kinds store a JSON projection in ShopDB and re-render to their native format on download; opaque vendor formats stay on the SFLD share with ShopDB holding metadata and the UNC pointer.", + "author": "ShopDB Team", + "dependencies": [], + "core_version": ">=0.16.0,<1.0.0", + "api_prefix": "/api/backups", + "provides": { + "features": [ + "configuration-backups" + ] + } +} diff --git a/plugins/backups/migrations/env.py b/plugins/backups/migrations/env.py new file mode 100644 index 0000000..a260bd4 --- /dev/null +++ b/plugins/backups/migrations/env.py @@ -0,0 +1,16 @@ +"""Alembic environment for the backups plugin migration chain. + +Delegates to the shared runner in shopdb.plugins.alembic_template, which filters +the metadata to this plugin's tables and drives Alembic against the per-plugin +version table alembic_version_backups. See ADR-008 for the ownership +model. Unlike the ten cutover plugins whose 0001 is a no-op anchor, this plugin +is NEW: its 0001 baseline really CREATES its tables, because the core chain +never built them. +""" +import os + +os.environ['PLUGIN_NAME'] = 'backups' + +from shopdb.plugins.alembic_template import run_migrations # noqa: E402 + +run_migrations() diff --git a/plugins/backups/migrations/script.py.mako b/plugins/backups/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/plugins/backups/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/plugins/backups/migrations/versions/0001_backups_baseline.py b/plugins/backups/migrations/versions/0001_backups_baseline.py new file mode 100644 index 0000000..885a3ca --- /dev/null +++ b/plugins/backups/migrations/versions/0001_backups_baseline.py @@ -0,0 +1,61 @@ +"""backups plugin baseline (real create). + +Built after the ADR-008 ownership cutover, so unlike the ten cutover plugins +whose 0001 is a stamp-only anchor, this baseline genuinely CREATES the table. +The core chain never knew about backuprevisions, so this per-plugin chain is its +sole authoritative creator. + +Emits explicit Alembic ops rather than using the create_plugin_tables helper: +the helper builds a per-plugin MetaData filtered to the plugin's own tables, so +the foreign key to the core assets table cannot resolve at CreateTable-compile +time (NoReferencedTableError). Same shape autogenerate produces. Tables inherit +the connection's default charset, matching how the core chain creates its own. +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'backups0001baseline' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'backuprevisions', + sa.Column('backuprevisionid', sa.Integer(), nullable=False), + sa.Column('assetid', sa.Integer(), nullable=False), + sa.Column('backupkind', sa.String(length=50), nullable=False), + sa.Column('storagebackend', sa.String(length=20), nullable=False, + server_default='shopdb'), + sa.Column('contenthash', sa.String(length=64), nullable=False), + sa.Column('payloadjson', sa.Text(length=16777215), nullable=True), + sa.Column('sharepath', sa.String(length=500), nullable=True), + sa.Column('sourcefilename', sa.String(length=255), nullable=True), + sa.Column('bytesize', sa.Integer(), nullable=True), + sa.Column('sourcehostname', sa.String(length=255), nullable=True), + sa.Column('collectedat', sa.DateTime(), nullable=True), + sa.Column('createdat', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['assetid'], ['assets.assetid'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('backuprevisionid'), + ) + op.create_index('ixbackuprevisionsassetid', 'backuprevisions', ['assetid']) + op.create_index('ixbackuprevisionsbackupkind', 'backuprevisions', + ['backupkind']) + op.create_index('ixbackuprevisionscontenthash', 'backuprevisions', + ['contenthash']) + # The dedup read path is "latest revision for this asset+kind", so the + # composite index is the one that actually gets used on every collector post. + op.create_index('ixbackuprevisionsassetkind', 'backuprevisions', + ['assetid', 'backupkind']) + + +def downgrade(): + op.drop_index('ixbackuprevisionsassetkind', table_name='backuprevisions') + op.drop_index('ixbackuprevisionscontenthash', table_name='backuprevisions') + op.drop_index('ixbackuprevisionsbackupkind', table_name='backuprevisions') + op.drop_index('ixbackuprevisionsassetid', table_name='backuprevisions') + op.drop_table('backuprevisions') diff --git a/plugins/backups/models/__init__.py b/plugins/backups/models/__init__.py new file mode 100644 index 0000000..0c5ad86 --- /dev/null +++ b/plugins/backups/models/__init__.py @@ -0,0 +1,5 @@ +"""Backups plugin models.""" + +from .backup import BackupRevision + +__all__ = ['BackupRevision'] diff --git a/plugins/backups/models/backup.py b/plugins/backups/models/backup.py new file mode 100644 index 0000000..6dfb50e --- /dev/null +++ b/plugins/backups/models/backup.py @@ -0,0 +1,123 @@ +"""Backup revision model. + +One row per DISTINCT configuration snapshot of an asset. The collector runs +every GE-Enforce cycle across the whole fleet, so the write path dedupes on +contenthash: a row appears only when a setting actually changed. That is what +turns a high-frequency collector into a readable revision history. + +Two storage backends, chosen by the kind (see services/registry.py): + + 'shopdb' Parsed structured config lives in payloadjson. The original file + is not kept because it re-renders exactly from the projection + (NTLARS .reg is the motivating case), which also lets one stored + revision render into more than one dialect on download. + + 'share' Opaque vendor formats that have no useful JSON representation + (part marker files and similar). Bytes stay on the SFLD share and + the row carries sharepath plus enough metadata to list, dedupe and + link to them. ShopDB never needs to parse these. + +contenthash is sha256 over the canonical form of whatever is authoritative for +the backend: the canonical JSON for 'shopdb', the raw file bytes for 'share'. +""" + +import json +from datetime import datetime + +from shopdb.api import db + + +class BackupRevision(db.Model): + """A single point-in-time configuration snapshot of an asset.""" + + __tablename__ = 'backuprevisions' + + backuprevisionid = db.Column(db.Integer, primary_key=True) + + # The asset the config BELONGS to, which is not always the asset it was + # collected from. NTLARS settings live in the controlling PC's registry but + # describe the machine, so the collector reports a machine number and the + # kind resolves it to the machine's asset. sourcehostname records the PC it + # actually came off. + assetid = db.Column( + db.Integer, + db.ForeignKey('assets.assetid', ondelete='CASCADE'), + nullable=False, + index=True, + ) + + backupkind = db.Column(db.String(50), nullable=False, index=True) + storagebackend = db.Column(db.String(20), nullable=False, default='shopdb') + + # sha256 of the canonical authoritative form. Dedup key together with + # (assetid, backupkind). + contenthash = db.Column(db.String(64), nullable=False, index=True) + + # Populated for storagebackend='shopdb' only. MEDIUMTEXT holding serialized + # JSON rather than a native JSON column, matching geenforce.manifestjson. + # Text also means the bytes come back exactly as written, so the canonical + # key ordering the codec produces survives the round trip - a native JSON + # column would renormalize it and make diffs between revisions unstable. + payloadjson = db.Column(db.Text(length=16777215), nullable=True) + + # Populated for storagebackend='share' only. Full UNC path. + sharepath = db.Column(db.String(500), nullable=True) + + # Original filename incl. extension. Vendor tools reject a renamed file, so + # downloads hand back exactly this name. + sourcefilename = db.Column(db.String(255), nullable=True) + bytesize = db.Column(db.Integer, nullable=True) + + # Which PC produced it, and when it was read off that PC (not when ShopDB + # stored it - a share-drop fallback can arrive much later). + sourcehostname = db.Column(db.String(255), nullable=True) + collectedat = db.Column(db.DateTime, nullable=True) + + createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + + __table_args__ = ( + db.Index('ixbackuprevisionsassetkind', 'assetid', 'backupkind'), + ) + + @property + def payload(self): + """Decoded payloadjson, or None for opaque (share-backed) kinds.""" + if not self.payloadjson: + return None + try: + return json.loads(self.payloadjson) + except (ValueError, TypeError) as exc: + # Do NOT return None here. A corrupt row would then look identical + # to an opaque share-backed revision, and the download route would + # fail somewhere further along with an unrelated error. Name the + # actual problem and the row it is in. + raise ValueError( + 'backuprevision {} has unreadable payloadjson: {}'.format( + self.backuprevisionid, exc)) + + @payload.setter + def payload(self, value): + if value is None: + self.payloadjson = None + else: + self.payloadjson = json.dumps(value, sort_keys=True, + separators=(',', ':')) + + def to_dict(self, includepayload=False): + data = { + 'backuprevisionid': self.backuprevisionid, + 'assetid': self.assetid, + 'backupkind': self.backupkind, + 'storagebackend': self.storagebackend, + 'contenthash': self.contenthash, + 'shorthash': (self.contenthash or '')[:12], + 'sharepath': self.sharepath, + 'sourcefilename': self.sourcefilename, + 'bytesize': self.bytesize, + 'sourcehostname': self.sourcehostname, + 'collectedat': self.collectedat.isoformat() if self.collectedat else None, + 'createdat': self.createdat.isoformat() if self.createdat else None, + } + if includepayload: + data['payloadjson'] = self.payload + return data diff --git a/plugins/backups/plugin.py b/plugins/backups/plugin.py new file mode 100644 index 0000000..96a978c --- /dev/null +++ b/plugins/backups/plugin.py @@ -0,0 +1,332 @@ +"""Backups plugin main class. + +Per-asset configuration backups with revision history, pluggable by kind +(services/registry.py). Accepts collector input over HTTPS per ADR-006 and +renders one asset panel per kind via the ADR-010 hook. + +Dedup is the load-bearing behaviour: GE-Enforce runs the collector every cycle +across the fleet, so apply_collector_payload inserts a revision only when the +content hash differs from that asset's latest for that kind. Without it a +readable history would be buried under millions of identical rows. +""" + +import base64 +import json +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional, Type + +from flask import Flask, Blueprint + +from shopdb.plugins.base import BasePlugin, PluginMeta + +from .api import backups_bp +from .models import BackupRevision +from .services.registry import (REGISTRY, getkind, canonicalhash, + DEFAULTSHAREROOT) +from .services.retention import prune + +logger = logging.getLogger(__name__) + + +class BackupsPlugin(BasePlugin): + """Configuration backup + revision history plugin.""" + + def __init__(self): + self._manifest = self._load_manifest() + + def _load_manifest(self) -> Dict: + manifest_path = Path(__file__).parent / 'manifest.json' + if manifest_path.exists(): + with open(manifest_path, 'r') as f: + return json.load(f) + return {} + + @property + def meta(self) -> PluginMeta: + return PluginMeta( + name=self._manifest.get('name', 'backups'), + version=self._manifest.get('version', '1.0.0'), + description=self._manifest.get('description', 'Configuration backups'), + author=self._manifest.get('author', 'ShopDB Team'), + dependencies=self._manifest.get('dependencies', []), + core_version=self._manifest.get('core_version', '>=0.2.0,<1.0.0'), + api_prefix=self._manifest.get('api_prefix', '/api/backups'), + ) + + def get_blueprint(self) -> Optional[Blueprint]: + return backups_bp + + def get_models(self) -> List[Type]: + return [BackupRevision] + + def get_permissions(self) -> List: + """RBAC for the human-facing routes. + + Deliberately split: download is separate from view because a .reg is a + restorable artifact, so a read-only auditor should be able to see that a + machine's config changed without being able to pull a file that + reconfigures it. + + No delete permission. Revisions are removed only by the retention + policy; a hand-delete route would let someone quietly drop the evidence + of a config change, and there is no operational need for it. Collector + INGEST is not covered here at all - that path authorizes on a + collector.ingest-scoped service token, not a user JWT. + """ + return [ + ('backups.view', 'View configuration backups', 'backups'), + ('backups.download', 'Download configuration backups', 'backups'), + ] + + def get_asset_panels(self) -> List[Dict]: + """One panel per kind, plus the DNC Info card. + + DNC Info sits ABOVE the history panels (lower position) because it + answers the question a tech actually arrives with - what is this + machine's controller talking to - while the history panels are for the + rarer restore case. + """ + panels = [kind.infopanel() for kind in REGISTRY.values() + if kind.infopanel()] + for position, kind in enumerate(REGISTRY.values()): + panels.append({ + 'id': 'backups-{}'.format(kind.key), + 'title': kind.displayname, + 'assettypes': list(kind.assettypes), + 'endpoint': '/api/backups/asset/{assetid}?kind=' + kind.key, + 'render': 'list', + 'map': { + 'title': 'label', + 'meta': [ + {'key': 'collectedat', 'label': 'Captured', 'format': 'date'}, + {'key': 'sourcehostname', 'label': 'From'}, + {'key': 'shorthash', 'label': 'Hash', 'mono': True}, + ], + }, + 'empty': kind.emptytext, + # The generic renderer has no per-item actions, and downloading + # a .reg needs a per-revision control plus a dialect choice, so + # the panel links to the plugin's own history view instead. + 'manage': { + 'to': '/backups/asset/{assetid}', + 'label': 'History / download', + 'emptylabel': 'View history', + }, + 'position': 40 + position, + }) + return panels + + def get_settings_defaults(self) -> List[Dict]: + """Schedule + retention, read by the collector and the prune job. + + The interval lives here rather than in the manifest entry because + GE-Enforce fires every cycle (several times a day on a shift change). + The collector reads backups_intervalhours and no-ops when its marker + file is younger than that, so the cadence is changed centrally in + ShopDB instead of by editing a script on the share. + """ + return [ + { + 'key': 'backups_intervalhours', + 'value': '24', + 'valuetype': 'integer', + 'category': 'backups', + 'description': 'Minimum hours between collection attempts on a ' + 'PC. GE-Enforce runs every cycle; the collector ' + 'skips until this much time has passed.', + }, + { + 'key': 'backups_shareroot', + 'value': DEFAULTSHAREROOT, + 'valuetype': 'string', + 'category': 'backups', + 'description': 'UNC root that opaque (non-JSON) backups are ' + 'written under by the collecting PC. Site-specific.', + }, + { + 'key': 'backups_retentioncount', + 'value': '50', + 'valuetype': 'integer', + 'category': 'backups', + 'description': 'Distinct revisions kept per asset per kind. ' + '0 keeps everything. The newest and the oldest ' + 'are never pruned.', + }, + { + 'key': 'backups_retentiondays', + 'value': '0', + 'valuetype': 'integer', + 'category': 'backups', + 'description': 'Also prune revisions older than this many days. ' + '0 disables age-based pruning.', + }, + ] + + # ---- ADR-006 collector contract ------------------------------------- + + def get_collector_schema(self) -> Optional[dict]: + return { + 'identityfield': 'machinenumber', + 'fields': { + 'type': 'object', + 'required': ['machinenumber', 'backupkind'], + 'properties': { + 'machinenumber': { + 'type': 'string', + 'description': 'Machine number the config belongs to. ' + 'Resolved against assets.assetnumber.', + }, + 'backupkind': { + 'type': 'string', + 'enum': sorted(REGISTRY.keys()), + }, + 'contentbase64': { + 'type': 'string', + 'description': "Raw backup file, base64. Required for " + "'shopdb' kinds (parsed server-side).", + }, + 'contenthash': { + 'type': 'string', + 'description': "sha256 of the file. Required for 'share' " + "kinds, whose bytes stay on the SFLD share.", + }, + 'sharepath': { + 'type': 'string', + 'description': "Full UNC path. Required for 'share' kinds.", + }, + 'sourcefilename': {'type': 'string'}, + 'sourcehostname': { + 'type': 'string', + 'description': 'PC the backup was read from. For NTLARS ' + 'this is the controlling PC, not the machine.', + }, + 'collectedat': {'type': 'string', 'format': 'date-time'}, + 'bytesize': {'type': 'integer'}, + }, + }, + } + + def apply_collector_payload(self, payload: dict) -> dict: + from shopdb.api import db + + warnings = [] + kindkey = (payload.get('backupkind') or '').strip().lower() + kind = getkind(kindkey) + if kind is None: + raise ValueError('unknown backupkind: {!r}'.format(kindkey)) + + assetid, err = kind.resolveassetid(payload) + if assetid is None: + # Unresolvable is reported, never silently dropped - an unmatched + # machine number means a real gap between the fleet and ShopDB. + raise ValueError('could not resolve asset: {}'.format(err)) + + projection = None + sharepath = None + sourcefilename = payload.get('sourcefilename') + bytesize = payload.get('bytesize') + + if kind.storagebackend == 'shopdb': + encoded = payload.get('contentbase64') + if not encoded: + raise ValueError( + "contentbase64 is required for kind '{}'".format(kindkey)) + raw = base64.b64decode(encoded) + bytesize = bytesize or len(raw) + projection = kind.parse(raw) + contenthash = canonicalhash(projection) + + # The fleet's machine number (pc-config.txt) is authoritative for + # WHICH asset this belongs to, but NTLARS carries its own MachineNo. + # A disagreement means the PC is running another machine's config - + # worth surfacing rather than quietly filing under the reported one. + embedded = getattr(kind, 'embeddedmachineno', None) + if embedded is not None: + embeddedno = embedded(projection) + reported = (payload.get('machinenumber') or '').strip() + if embeddedno and reported and embeddedno != reported: + warnings.append( + 'reported machine number {!r} but NTLARS is configured ' + 'for {!r}'.format(reported, embeddedno)) + else: + contenthash = (payload.get('contenthash') or '').strip().lower() + sharepath = (payload.get('sharepath') or '').strip() + if not contenthash or not sharepath: + raise ValueError( + "contenthash and sharepath are required for kind '{}'".format( + kindkey)) + + # with_for_update serializes concurrent posts for the same asset+kind. + # The whole fleet collects on the same GE-Enforce cycle, so a retry or + # two PCs claiming one machine number can otherwise both pass the hash + # check and insert duplicate identical revisions. No unique constraint + # can cover this: dedup is against the LATEST row only, because a config + # reverting to an earlier state is a legitimate new revision. + latest = (db.session.query(BackupRevision) + .filter(BackupRevision.assetid == assetid, + BackupRevision.backupkind == kindkey) + .order_by(BackupRevision.backuprevisionid.desc()) + .with_for_update() + .first()) + + if latest is not None and latest.contenthash == contenthash: + return { + 'action': 'noop', + 'assetid': assetid, + 'backuprevisionid': latest.backuprevisionid, + 'warnings': warnings, + } + + collectedat = None + rawcollected = payload.get('collectedat') + if rawcollected: + try: + collectedat = datetime.fromisoformat(rawcollected.replace('Z', '+00:00')) + if collectedat.tzinfo is not None: + # Convert, don't just drop the offset: createdat is utcnow(), + # so keeping local wall time would skew a -04:00 bay by 4h + # against its own history. + collectedat = collectedat.astimezone(timezone.utc).replace( + tzinfo=None) + except ValueError: + warnings.append('unparseable collectedat: {!r}'.format(rawcollected)) + + revision = BackupRevision( + assetid=assetid, + backupkind=kindkey, + storagebackend=kind.storagebackend, + contenthash=contenthash, + sharepath=sharepath, + sourcefilename=sourcefilename, + bytesize=bytesize, + sourcehostname=payload.get('sourcehostname'), + collectedat=collectedat or datetime.utcnow(), + ) + revision.payload = projection + db.session.add(revision) + # flush, not commit: the collector dispatcher owns the transaction and + # commits after writing its AuditLog row. Committing here would leave an + # unaudited revision behind if that write then failed. + db.session.flush() + + pruned = prune( + assetid, kindkey, + retentioncount=self.get_setting('backups_retentioncount', 0), + retentiondays=self.get_setting('backups_retentiondays', 0), + ) + if pruned: + warnings.append('pruned {} old revision(s) per retention policy' + .format(pruned)) + + return { + 'action': 'created', + 'assetid': assetid, + 'backuprevisionid': revision.backuprevisionid, + 'warnings': warnings, + } + + def init_app(self, app: Flask, db_instance) -> None: + logger.info('Backups plugin initialized (v%s, kinds: %s)', + self.meta.version, ', '.join(sorted(REGISTRY))) diff --git a/plugins/backups/services/__init__.py b/plugins/backups/services/__init__.py new file mode 100644 index 0000000..7563f19 --- /dev/null +++ b/plugins/backups/services/__init__.py @@ -0,0 +1,6 @@ +"""Backups plugin services: kind registry and per-kind codecs.""" + +from .registry import REGISTRY, BackupKind, getkind, canonicalhash, byteshash, DEFAULTSHAREROOT + +__all__ = ['REGISTRY', 'BackupKind', 'getkind', 'canonicalhash', 'byteshash', + 'DEFAULTSHAREROOT'] diff --git a/plugins/backups/services/dncinfo.py b/plugins/backups/services/dncinfo.py new file mode 100644 index 0000000..e1333a3 --- /dev/null +++ b/plugins/backups/services/dncinfo.py @@ -0,0 +1,161 @@ +"""DNC Info card. + +Surfaces the handful of NTLARS settings a tech actually asks about on the +machine's page, so the common question ("what is this machine's controller +talking to?") is answered without downloading and reading a .reg. + +Sections, drawn from the machine's LATEST ntlars revision: + + eFocas Fanuc ethernet link - IP, socket, dual-path. Present on 143 of the + 147 known-good backups, so it is shown whenever it has content. + Serial RS-232 link parameters. Always populated (Baud, Data Bits and + friends carry defaults even where the link is unused), so it is + always shown. + NTSHR Network share the controller pulls programs from. Populated on only + 18 of 147, hence the has-content gate: showing an empty NTSHR block + on 129 machines would be noise. + MARK Part-marker settings. Gated on the ASSET being a Part Marker in + ShopDB, NOT on the key having content. + +WHY MARK IS GATED ON THE ASSET, NOT THE KEY: + The obvious rule - show MARK when it has content - does not work. MARK is + populated on 145 of 147 machines because Baud/Data Bits carry serial + defaults everywhere, and the fields that would identify a marker + (CageCode, DataHost, DataPath, MarkMasterPath) are empty across the entire + corpus. The one field that is set, DncPatterns, reads YES on 103 of 147 + including ordinary lathes, so it is a DNC pattern-matching option and not a + marker flag. No value in the DNC tree distinguishes a part marker, so the + machine's type in ShopDB is the only reliable signal. +""" + +# Values equal to these (case-insensitively) count as "no content". +EMPTYISH = ('', '0', 'no', 'none') + +DEFAULTPARTMARKERTYPES = ('Part Marker',) + + +def _keyvalues(projection, path): + """Values under one subkey of the stored projection, or {}.""" + for key in (projection or {}).get('keys', []): + if (key.get('path') or '').lower() == path.lower(): + return key.get('values') or {} + return {} + + +def _hascontent(values): + """True when at least one value carries something meaningful.""" + for entry in values.values(): + data = entry.get('data') + if isinstance(data, (list, tuple)): + if any(str(x).strip() for x in data): + return True + continue + if str(data).strip().lower() not in EMPTYISH: + return True + return False + + +def _fields(values, mono=()): + """Render a subkey's values as keyvalue panel fields, empties dropped.""" + out = [] + for name in sorted(values): + data = values[name].get('data') + if isinstance(data, (list, tuple)): + data = ', '.join(str(x) for x in data) + text = '' if data is None else str(data) + if not text.strip(): + continue + out.append({ + 'label': name, + 'value': text, + 'mono': name in mono, + }) + return out + + +def ispartmarker(assetid, typenames=None): + """True when this asset is a Part Marker according to ShopDB. + + Reads the machines plugin defensively: a lean per-site build (ADR-014) may + not install it, and the DNC Info card must degrade to "no MARK section" + rather than erroring the whole panel. + """ + typenames = tuple(t.lower() for t in (typenames or DEFAULTPARTMARKERTYPES)) + try: + from shopdb.api import db + from plugins.machines.models import Machine, MachineType + except ImportError: + return False + try: + row = (db.session.query(MachineType.machinetype) + .join(Machine, Machine.machinetypeid == MachineType.machinetypeid) + .filter(Machine.assetid == assetid) + .first()) + except Exception: + return False + return bool(row) and (row[0] or '').strip().lower() in typenames + + +def _cncismarker(projection): + """True when NTLARS itself says the controller is a marker. + + General\\Cnc reads 'MARKER' on the part markers (0600 and 0614 in the + known-good corpus) and a controller family - Fanuc 30, Fanuc 16, OKUMA, + Fidia - everywhere else. This is the one place the DNC tree does + distinguish a marker; nothing inside the MARK key does, since MARK carries + serial defaults on nearly every machine. + """ + general = _keyvalues(projection, 'General') + cnc = str((general.get('Cnc') or {}).get('data') or '').strip().upper() + return cnc == 'MARKER' + + +def build(projection, assetid, partmarkertypes=None): + """Build the DNC Info card payload from a stored ntlars projection. + + Returns {'fields': [...]} in the shape the generic keyvalue renderer wants, + with section headings inlined as labelless separators. + """ + sections = [] + + # General first: what the controller IS, before what it talks to. Cnc gives + # the controller family, NcIF the interface actually in use (EFOCAS on 127 + # of 147, NTSHR on 16, SERIAL on 3, HSSB on 1), HostType the DNC host. + general = _keyvalues(projection, 'General') + wanted = ('Cnc', 'NcIF', 'HostType') + generalfields = _fields( + {n: v for n, v in general.items() if n in wanted}) + if generalfields: + sections.append(('General', generalfields)) + + efocas = _keyvalues(projection, 'eFocas') + if _hascontent(efocas): + sections.append(('eFocas (ethernet link)', _fields(efocas, mono=('IpAddr',)))) + + serial = _keyvalues(projection, 'Serial') + if serial: + sections.append(('Serial (RS-232)', _fields(serial))) + + ntshr = _keyvalues(projection, 'NTSHR') + if _hascontent(ntshr): + sections.append(('NTSHR (program share)', + _fields(ntshr, mono=('ShrFolder', 'ShrFolder2', + 'ShrFolder3', 'ShrHost')))) + + # Either signal is enough: NTLARS's own Cnc=MARKER works before anyone has + # set the machine's type in ShopDB and on lean builds with no machines + # plugin, while the ShopDB type still covers a marker whose Cnc says + # something else. + if _cncismarker(projection) or ispartmarker(assetid, partmarkertypes): + mark = _keyvalues(projection, 'MARK') + if mark: + sections.append(('MARK (part marker)', + _fields(mark, mono=('DataPath', 'MarkMasterPath')))) + + fields = [] + for title, entries in sections: + if not entries: + continue + fields.append({'label': title, 'value': '', 'heading': True}) + fields.extend(entries) + return {'fields': fields, 'sectioncount': len(sections)} diff --git a/plugins/backups/services/ntlars.py b/plugins/backups/services/ntlars.py new file mode 100644 index 0000000..593a75c --- /dev/null +++ b/plugins/backups/services/ntlars.py @@ -0,0 +1,308 @@ +"""NTLARS / DNC registry backup codec. + +Converts between Windows .reg files and a dialect-neutral JSON projection. + +WHY DIALECT-NEUTRAL: NTLARS is a 32-bit app, so its settings physically live +under HKLM\\SOFTWARE\\WOW6432Node\\GE Aircraft Engines\\DNC. But NTLARS's own +Save... button exports them WITHOUT the WOW6432Node segment (it writes the path +it asks for, before the WOW64 redirector rewrites it). Both dialects therefore +exist in the wild: + + NTLARS Save... output HKLM\\SOFTWARE\\GE Aircraft Engines\\DNC + scripted / reg export HKLM\\SOFTWARE\\WOW6432Node\\GE Aircraft Engines\\DNC + +Parsing strips whichever root matched and stores subkeys RELATIVE to it, so the +stored revision commits to neither. render() then re-attaches whichever root the +consumer needs: + + dialect='ntlars' no WOW6432Node - what the NTLARS Load... button expects + dialect='wow6432node' explicit - what `reg import` needs on a 64-bit box + +Getting this backwards is silent: a reg import of the NTLARS dialect on 64-bit +writes to the 64-bit hive, where NTLARS will never look, and reports success. + +CANONICAL ORDERING: keys and value names are sorted on parse. MySQL's JSON type +normalizes object key order anyway, so preserving source order is not possible +end-to-end; sorting makes it deterministic instead, which is what makes diffs +between revisions stable. Re-rendered files are semantically identical to their +source, not byte-identical. +""" + +import re + +SCHEMA = 'ntlars/1' + +REGROOT = 'HKEY_LOCAL_MACHINE' +DNCPATH = r'SOFTWARE\GE Aircraft Engines\DNC' +DNCPATHWOW = r'SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC' + +ROOTNTLARS = '{}\\{}'.format(REGROOT, DNCPATH) +ROOTWOW = '{}\\{}'.format(REGROOT, DNCPATHWOW) + +# Order is not significant: the two roots diverge immediately after +# 'SOFTWARE\\' (GE vs WOW), so neither is a string prefix of the other and +# _striproot's startswith test cannot match the wrong one. Listed longest-first +# only for readability. +KNOWNROOTS = (ROOTWOW, ROOTNTLARS) + +HEADER = 'Windows Registry Editor Version 5.00' + +# hex(N): type codes that appear in .reg files, mapped to registry type names. +HEXTYPES = { + 0: 'REG_NONE', + 1: 'REG_SZ', + 2: 'REG_EXPAND_SZ', + 3: 'REG_BINARY', + 4: 'REG_DWORD', + 7: 'REG_MULTI_SZ', + 11: 'REG_QWORD', +} +HEXTYPECODES = {v: k for k, v in HEXTYPES.items()} + + +class NtlarsParseError(ValueError): + """Raised when input is not a .reg file we can make sense of.""" + + +def decodereg(raw): + """Decode .reg bytes to text. + + .reg files are conventionally UTF-16LE with a BOM (that is what both + regedit and NTLARS emit), but hand-edited ones show up as UTF-8. Sniff the + BOM rather than trusting the extension. + """ + if isinstance(raw, str): + return raw + if raw.startswith(b'\xff\xfe'): + return raw.decode('utf-16-le')[1:] + if raw.startswith(b'\xfe\xff'): + return raw.decode('utf-16-be')[1:] + if raw.startswith(b'\xef\xbb\xbf'): + return raw.decode('utf-8-sig') + # No BOM. UTF-16LE ASCII text has a NUL in every other byte. + if b'\x00' in raw[:64]: + return raw.decode('utf-16-le', errors='replace') + return raw.decode('utf-8', errors='replace') + + +def _unescape(s): + return s.replace('\\\\', '\x00').replace('\\"', '"').replace('\x00', '\\') + + +def _escape(s): + return s.replace('\\', '\\\\').replace('"', '\\"') + + +def _joincontinuations(text): + """Fold .reg line continuations (trailing backslash) into single lines.""" + out = [] + for line in text.replace('\r\n', '\n').replace('\r', '\n').split('\n'): + if out and out[-1].endswith('\\'): + out[-1] = out[-1][:-1] + line.strip() + else: + out.append(line) + return out + + +def _parsehexvalue(body): + """Parse the body of a hex:/hex(N): value into (typename, data).""" + m = re.match(r'^hex(?:\((?P[0-9a-fA-F]+)\))?:(?P.*)$', body, re.S) + if not m: + raise NtlarsParseError('unparseable hex value: {!r}'.format(body)) + code = int(m.group('code'), 16) if m.group('code') else 3 + tokens = [t.strip() for t in m.group('bytes').split(',') if t.strip()] + try: + data = bytes(int(t, 16) for t in tokens) + except ValueError as exc: + raise NtlarsParseError('bad hex byte in value: {}'.format(exc)) + + typename = HEXTYPES.get(code, 'REG_BINARY') + + # Wide-string hex types decode back to text so diffs stay readable. + if typename in ('REG_SZ', 'REG_EXPAND_SZ'): + return typename, data.decode('utf-16-le', errors='replace').rstrip('\x00') + if typename == 'REG_MULTI_SZ': + text = data.decode('utf-16-le', errors='replace') + return typename, [p for p in text.split('\x00') if p] + # REG_QWORD must come back as an int: _rendervalue turns it back into + # little-endian bytes via int(), so storing the "aa,bb" byte form here + # would raise at download time - i.e. precisely when someone is trying to + # restore a machine. + if typename == 'REG_QWORD': + return typename, int.from_bytes(data, 'little') + return typename, ','.join('{:02x}'.format(b) for b in data) + + +def _parsevalue(body): + """Parse the right-hand side of a .reg value assignment.""" + body = body.strip() + if body.startswith('"'): + if not body.endswith('"') or len(body) < 2: + raise NtlarsParseError('unterminated string value: {!r}'.format(body)) + return 'REG_SZ', _unescape(body[1:-1]) + if body.lower().startswith('dword:'): + try: + return 'REG_DWORD', int(body.split(':', 1)[1].strip(), 16) + except ValueError: + raise NtlarsParseError('bad dword value: {!r}'.format(body)) + if body.lower().startswith('hex'): + return _parsehexvalue(body) + if body == '-': + return 'DELETE', None + raise NtlarsParseError('unrecognised value form: {!r}'.format(body)) + + +def _striproot(keypath): + """Strip a known DNC root, returning the relative subkey path. + + Returns None for keys outside the DNC tree so callers can ignore them + rather than silently folding unrelated hives into the backup. + """ + for root in KNOWNROOTS: + if keypath.upper() == root.upper(): + return '' + prefix = root.upper() + '\\' + if keypath.upper().startswith(prefix): + return keypath[len(prefix):] + return None + + +def parse(raw): + """Parse .reg bytes/text into the dialect-neutral JSON projection. + + Returns {'schema', 'sourcedialect', 'keys': [{'path', 'values': {...}}]} + with keys and value names sorted for deterministic diffing. + """ + text = decodereg(raw) + lines = _joincontinuations(text) + + if not any(line.strip().lower().startswith('windows registry editor') + or line.strip().lower().startswith('regedit4') + for line in lines[:5]): + raise NtlarsParseError('missing "Windows Registry Editor" header') + + keys = {} + current = None + sawwow = False + sawplain = False + + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith(';'): + continue + + if stripped.startswith('[') and stripped.endswith(']'): + keypath = stripped[1:-1].strip() + if keypath.startswith('-'): + current = None # key deletion, not a backup concern + continue + if keypath.upper().startswith(ROOTWOW.upper()): + sawwow = True + elif keypath.upper().startswith(ROOTNTLARS.upper()): + sawplain = True + rel = _striproot(keypath) + if rel is None: + current = None # outside the DNC tree - ignore + continue + current = rel + keys.setdefault(current, {}) + continue + + if current is None or '=' not in stripped: + continue + + # Match the QUOTED name and split at the '=' that follows its closing + # quote. A plain split('=', 1) breaks on any value name containing '=' + # or an escaped quote - both legal in the registry - and silently drops + # the value. + match = re.match(r'^(?:@|"((?:[^"\\]|\\.)*)")\s*=\s*(.*)$', stripped, re.S) + if not match: + continue + name = '' if match.group(1) is None else _unescape(match.group(1)) + body = match.group(2) + + # Deliberately NOT caught. A backup that silently dropped an + # unparseable value would present as complete and restore a machine + # with a setting missing - the exact silent-failure class this project + # has repeatedly been bitten by. Fail the whole parse instead; the + # collector reports it and the previous good revision stays newest. + typename, data = _parsevalue(body) + if typename == 'DELETE': + continue + keys[current][name] = {'type': typename, 'data': data} + + if not keys: + raise NtlarsParseError( + 'no keys under {} or {} - not an NTLARS DNC backup'.format( + ROOTNTLARS, ROOTWOW)) + + dialect = 'wow6432node' if sawwow else ('ntlars' if sawplain else 'unknown') + + return { + 'schema': SCHEMA, + 'sourcedialect': dialect, + 'keys': [ + {'path': path, 'values': dict(sorted(keys[path].items()))} + for path in sorted(keys) + ], + } + + +def _rendervalue(name, entry): + typename = entry.get('type', 'REG_SZ') + data = entry.get('data') + lhs = '@' if name == '' else '"{}"'.format(_escape(name)) + + if typename == 'REG_SZ': + return '{}="{}"'.format(lhs, _escape('' if data is None else str(data))) + if typename == 'REG_DWORD': + return '{}=dword:{:08x}'.format(lhs, int(data) & 0xFFFFFFFF) + if typename == 'REG_QWORD': + return '{}=hex(b):{}'.format(lhs, _hexbytes( + int(data).to_bytes(8, 'little'))) + if typename == 'REG_EXPAND_SZ': + payload = ('' if data is None else str(data)).encode('utf-16-le') + b'\x00\x00' + return '{}=hex(2):{}'.format(lhs, _hexbytes(payload)) + if typename == 'REG_MULTI_SZ': + parts = data if isinstance(data, list) else [str(data)] + payload = ''.join(p + '\x00' for p in parts).encode('utf-16-le') + b'\x00\x00' + return '{}=hex(7):{}'.format(lhs, _hexbytes(payload)) + + # REG_BINARY / REG_NONE: data is the "aa,bb,cc" form parse() produced. + raw = bytes(int(t, 16) for t in str(data).split(',') if t.strip()) if data else b'' + code = HEXTYPECODES.get(typename, 3) + prefix = 'hex:' if code == 3 else 'hex({:x}):'.format(code) + return '{}={}{}'.format(lhs, prefix, _hexbytes(raw)) + + +def _hexbytes(raw): + return ','.join('{:02x}'.format(b) for b in raw) + + +def render(projection, dialect='ntlars', comments=None): + """Render the JSON projection back to .reg bytes (UTF-16LE, BOM, CRLF). + + dialect='ntlars' omits WOW6432Node - use with the NTLARS Load... button + dialect='wow6432node' includes it - use with `reg import` on 64-bit + """ + if dialect not in ('ntlars', 'wow6432node'): + raise ValueError('unknown dialect: {!r}'.format(dialect)) + root = ROOTWOW if dialect == 'wow6432node' else ROOTNTLARS + + out = [HEADER, ''] + for line in (comments or []): + out.append('; {}'.format(line)) + if comments: + out.append('') + + for key in projection.get('keys', []): + path = key.get('path', '') + out.append('[{}]'.format(root + ('\\' + path if path else ''))) + for name, entry in (key.get('values') or {}).items(): + out.append(_rendervalue(name, entry)) + out.append('') + + text = '\r\n'.join(out) + if not text.endswith('\r\n'): + text += '\r\n' + return b'\xff\xfe' + text.encode('utf-16-le') diff --git a/plugins/backups/services/registry.py b/plugins/backups/services/registry.py new file mode 100644 index 0000000..449a907 --- /dev/null +++ b/plugins/backups/services/registry.py @@ -0,0 +1,237 @@ +"""Backup kind registry. + +A "kind" is one category of configuration backup (NTLARS/DNC registry, part +marker, UDC, CMM ...). Each kind declares three things the rest of the plugin +needs and nothing else: + + where its bytes live 'shopdb' (parsed into payloadjson) or 'share' + (opaque vendor file left on the SFLD share) + how to resolve an asset the collector reports an identifier; the kind turns + it into the assetid the revision belongs to + how to render downloads one or more output formats + +Adding a kind is a class plus one REGISTRY entry. Nothing else in the plugin +knows kind names. +""" + +import hashlib +import json + +from . import ntlars as ntlarscodec + +# Default root of the opaque-backup tree on the SFLD share. Connected PCs write +# here directly (they need SFLD creds - a SYSTEM process hitting a UNC path +# without them gets an access-denied that Test-Path reports as "not found"). +# +# This is the WEST JEFFERSON path and is only a DEFAULT: the live value is the +# backups_shareroot setting, because a bundled plugin in a multi-site product +# must not hardcode one site's file server. +DEFAULTSHAREROOT = r'\\tsgwp00525.wjs.geaerospace.net\shared\dt\shopfloor\backups' + + +def canonicalhash(projection): + """sha256 over the SEMANTIC content only. Dedup key for 'shopdb' kinds. + + Deliberately hashes just schema + keys, excluding sourcedialect. The same + settings exported through NTLARS's Save... button and through a scripted + reg export parse to identical keys but differing sourcedialect; hashing the + whole projection would record that as a change and produce a spurious + revision every time the collection route changed. Storage is + dialect-neutral, so the dedup key has to be too. + """ + subset = { + 'schema': projection.get('schema'), + 'keys': projection.get('keys'), + } + blob = json.dumps(subset, sort_keys=True, separators=(',', ':')) + return hashlib.sha256(blob.encode('utf-8')).hexdigest() + + +def byteshash(raw): + """sha256 over raw bytes. Dedup key for 'share' kinds.""" + return hashlib.sha256(raw).hexdigest() + + +class BackupKind: + """Base class. Subclasses override what applies to them.""" + + key = None + displayname = None + storagebackend = 'shopdb' + assettypes = ['*'] + # Human note shown on the panel when there is nothing yet. + emptytext = 'No backups on record.' + + def parse(self, raw): + """Opaque kinds return None; parseable kinds return the projection.""" + return None + + def formats(self): + """Downloadable formats: [{'id','label','ext','mimetype'}].""" + return [] + + def render(self, projection, formatid, comments=None): + """Return (bytes, extension, mimetype) for a 'shopdb' kind.""" + raise NotImplementedError + + def resolveassetid(self, payload): + """Map a collector payload to the assetid the backup belongs to.""" + raise NotImplementedError + + def infopanel(self): + """Optional 'at a glance' card built from this kind's LATEST revision. + + Returns an ADR-010 panel dict, or None when the kind has no summary + worth surfacing. Declared by the kind rather than hardcoded in the + plugin so a successor technology (NTLARS/DNC is expected to give way to + Shopfloor Connect) ships its own card by adding a class, without + touching the plugin or the panel wiring. + """ + return None + + def buildinfo(self, projection, assetid, **options): + """Build this kind's info-card payload. Only called when infopanel().""" + return {'fields': [], 'sectioncount': 0} + + def sharedir(self, machinetype, identifier, shareroot=None): + """UNC directory a 'share' kind's files are expected under. + + Advisory only - the authoritative path is the sharepath the collector + reports, since the PC is what actually wrote the file. This builds the + conventional location for display and for validating a reported path. + """ + return '{}\\{}\\{}\\{}'.format( + shareroot or DEFAULTSHAREROOT, + machinetype or 'unknown', identifier or 'unknown', self.key) + + +class NtlarsKind(BackupKind): + """NTLARS / DNC registry settings. + + Collected from the CONTROLLING PC's registry but belongs to the MACHINE: + the settings describe how to talk to that machine's controller, so they + follow the machine when a PC is swapped. resolveassetid therefore keys on + the reported machine number, not the hostname. + """ + + key = 'ntlars' + displayname = 'NTLARS / DNC Settings' + storagebackend = 'shopdb' + assettypes = ['machine'] + emptytext = 'No NTLARS settings captured yet.' + + @staticmethod + def embeddedmachineno(projection): + """The MachineNo NTLARS itself is configured with (General tab).""" + for key in (projection or {}).get('keys', []): + if (key.get('path') or '').lower() == 'general': + entry = (key.get('values') or {}).get('MachineNo') or {} + return str(entry.get('data') or '').strip() + return '' + + def parse(self, raw): + """Parse, then refuse to record an unconfigured NTLARS install. + + A freshly imaged PC opens NTLARS with a blank General tab (no MachineNo, + CNC, Host or Interface type) until a tech restores the config. Storing + that would make an empty config the newest revision at exactly the + moment someone needs the last good one - so a blank MachineNo is + rejected as "nothing worth backing up" rather than accepted as a change. + Two of the 320 known-good backups on the share already have this shape. + """ + projection = ntlarscodec.parse(raw) + if not self.embeddedmachineno(projection): + raise ValueError( + 'NTLARS General\\MachineNo is empty - unconfigured install, ' + 'refusing to record it as a revision') + return projection + + def formats(self): + return [ + { + 'id': 'ntlars', + 'label': 'NTLARS Load... (.reg)', + 'ext': '.reg', + 'mimetype': 'application/octet-stream', + 'hint': 'Restore with the Load... button in the NTLARS settings dialog.', + }, + { + 'id': 'wow6432node', + 'label': 'Direct reg import (.reg, WOW6432Node)', + 'ext': '.reg', + 'mimetype': 'application/octet-stream', + 'hint': 'Use when importing outside NTLARS on a 64-bit machine.', + }, + ] + + def render(self, projection, formatid, comments=None): + if formatid not in ('ntlars', 'wow6432node'): + raise ValueError('unknown format for ntlars kind: {}'.format(formatid)) + raw = ntlarscodec.render(projection, dialect=formatid, comments=comments) + return raw, '.reg', 'application/octet-stream' + + def infopanel(self): + return { + 'id': 'backups-dncinfo', + 'title': 'DNC Info', + 'assettypes': ['machine'], + 'endpoint': '/api/backups/asset/{assetid}/info?kind=ntlars', + 'render': 'keyvalue', + 'empty': 'No NTLARS settings captured for this machine yet.', + # Above the history panels: this answers the question a tech + # arrives with, while history is for the rarer restore case. + 'position': 38, + } + + def buildinfo(self, projection, assetid, **options): + from . import dncinfo + return dncinfo.build(projection, assetid, + partmarkertypes=options.get('partmarkertypes')) + + def resolveassetid(self, payload): + from shopdb.api import db, Asset + + machinenumber = (payload.get('machinenumber') or '').strip() + if not machinenumber: + return None, 'no machinenumber in payload' + asset = db.session.query(Asset).filter( + Asset.assetnumber == machinenumber).first() + if asset is None: + return None, 'no asset with assetnumber {!r}'.format(machinenumber) + return asset.assetid, None + + +class PartMarkerKind(BackupKind): + """Telesis part marker configuration. + + Opaque vendor filetype with no useful JSON representation, so ShopDB stores + metadata and a UNC pointer while the file itself stays on the share. The + original filename and extension are preserved because the vendor tool + rejects a renamed file. + """ + + key = 'partmarker' + displayname = 'Part Marker Configuration' + storagebackend = 'share' + assettypes = ['machine', 'measuring_tool'] + emptytext = 'No part marker backups on record.' + + def resolveassetid(self, payload): + from shopdb.api import db, Asset + + identifier = (payload.get('machinenumber') + or payload.get('assetnumber') or '').strip() + if not identifier: + return None, 'no machinenumber/assetnumber in payload' + asset = db.session.query(Asset).filter( + Asset.assetnumber == identifier).first() + if asset is None: + return None, 'no asset with assetnumber {!r}'.format(identifier) + return asset.assetid, None + + +REGISTRY = {k.key: k for k in (NtlarsKind(), PartMarkerKind())} + + +def getkind(key): + return REGISTRY.get((key or '').strip().lower()) diff --git a/plugins/backups/services/retention.py b/plugins/backups/services/retention.py new file mode 100644 index 0000000..35abf25 --- /dev/null +++ b/plugins/backups/services/retention.py @@ -0,0 +1,68 @@ +"""Revision retention. + +Dedup already keeps growth low - a revision appears only when a setting really +changed - so retention exists for the pathological case, not the normal one: a +value that flaps (or two PCs alternately claiming one machine number) would +otherwise append a revision every collection cycle forever. + +Two independent limits, both off by default at 0: + + retentioncount keep at most N revisions per asset per kind + retentiondays additionally drop anything older than N days + +The NEWEST and the OLDEST revision are never pruned. The newest is the one a +tech restores from; the oldest is the earliest known-good baseline, which is +usually the most valuable row in the table and the one a naive "keep last N" +would delete first. +""" + +from datetime import datetime, timedelta + +from shopdb.api import db + +from ..models import BackupRevision + + +def prune(assetid, backupkind, retentioncount=0, retentiondays=0): + """Delete surplus revisions for one asset+kind. Returns the number removed. + + Caller commits. Returns 0 when both limits are disabled. + """ + retentioncount = int(retentioncount or 0) + retentiondays = int(retentiondays or 0) + if retentioncount <= 0 and retentiondays <= 0: + return 0 + + revisions = (db.session.query(BackupRevision) + .filter(BackupRevision.assetid == assetid, + BackupRevision.backupkind == backupkind) + .order_by(BackupRevision.backuprevisionid.desc()) + .all()) + if len(revisions) <= 2: + return 0 # newest + oldest are both protected + + newest = revisions[0] + oldest = revisions[-1] + protected = {newest.backuprevisionid, oldest.backuprevisionid} + + doomed = [] + + if retentioncount > 0 and len(revisions) > retentioncount: + # Walk from the oldest END of the middle, so the surplus dropped is + # always the least recent, never the newest. + for revision in revisions[retentioncount:]: + if revision.backuprevisionid not in protected: + doomed.append(revision) + + if retentiondays > 0: + cutoff = datetime.utcnow() - timedelta(days=retentiondays) + for revision in revisions: + if revision.backuprevisionid in protected: + continue + stamp = revision.collectedat or revision.createdat + if stamp and stamp < cutoff and revision not in doomed: + doomed.append(revision) + + for revision in doomed: + db.session.delete(revision) + return len(doomed) diff --git a/scripts/import_ntlars_backups.py b/scripts/import_ntlars_backups.py new file mode 100644 index 0000000..11177e5 --- /dev/null +++ b/scripts/import_ntlars_backups.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +"""Seed backup history from a directory of existing NTLARS .reg files. + +The share already holds ~150 per-machine backups collected by hand over years +(S:\\DT\\RegFiles\\Dnc\\Backup Copies\\.reg and the pxe-images +mirrors). Importing them gives every machine a baseline revision on day one, +so the feature is useful before the fleet collector has run even once. + +Files are keyed by machine number in the FILENAME, which is how the share names +them. The MachineNo embedded in the file is compared against it and any +disagreement is reported: in the curated set the two always agree, so a +mismatch means the file is misfiled and should not be trusted as that machine's +baseline. + +Dry run by default. Nothing is written without --commit. + + venv/bin/python scripts/import_ntlars_backups.py /home/camp/pxe-images/ntlars-deploy + venv/bin/python scripts/import_ntlars_backups.py /path/to/regs --commit +""" + +import argparse +import os +import sys +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from shopdb import create_app # noqa: E402 +from shopdb.api import db # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory', help='directory holding .reg files') + parser.add_argument('--commit', action='store_true', + help='actually write (default is a dry run)') + parser.add_argument('--hostname', default='imported', + help="sourcehostname to record (default 'imported')") + args = parser.parse_args() + + if not os.path.isdir(args.directory): + parser.error('not a directory: {}'.format(args.directory)) + + regs = sorted(f for f in os.listdir(args.directory) if f.lower().endswith('.reg')) + if not regs: + parser.error('no .reg files in {}'.format(args.directory)) + + app = create_app() + with app.app_context(): + from shopdb.core.models import Asset + from plugins.backups.models import BackupRevision + from plugins.backups.services.registry import getkind, canonicalhash + + kind = getkind('ntlars') + + created = skipped = unresolved = rejected = duplicate = 0 + mismatches = [] + + for filename in regs: + path = os.path.join(args.directory, filename) + machinenumber = os.path.splitext(filename)[0] + + with open(path, 'rb') as handle: + raw = handle.read() + + try: + projection = kind.parse(raw) + except ValueError as exc: + # Unconfigured NTLARS installs are in the corpus; they are not + # a usable baseline for anyone. + print(' REJECT {:<12} {}'.format(machinenumber, exc)) + rejected += 1 + continue + + embedded = kind.embeddedmachineno(projection) + if embedded and embedded != machinenumber: + mismatches.append((machinenumber, embedded)) + + asset = db.session.query(Asset).filter( + Asset.assetnumber == machinenumber).first() + if asset is None: + print(' NOASSET {:<12} no asset with that assetnumber'.format( + machinenumber)) + unresolved += 1 + continue + + contenthash = canonicalhash(projection) + latest = (db.session.query(BackupRevision) + .filter(BackupRevision.assetid == asset.assetid, + BackupRevision.backupkind == 'ntlars') + .order_by(BackupRevision.backuprevisionid.desc()) + .first()) + if latest is not None and latest.contenthash == contenthash: + duplicate += 1 + continue + + if args.commit: + revision = BackupRevision( + assetid=asset.assetid, + backupkind='ntlars', + storagebackend='shopdb', + contenthash=contenthash, + sourcefilename=filename, + bytesize=len(raw), + sourcehostname=args.hostname, + # The file's mtime is the closest thing to when the config + # was actually captured; createdat records the import. + collectedat=datetime.utcfromtimestamp(os.path.getmtime(path)), + ) + revision.payload = projection + db.session.add(revision) + created += 1 + + if args.commit: + db.session.commit() + + print() + print('files : {}'.format(len(regs))) + print('would create : {}'.format(created) if not args.commit + else 'created : {}'.format(created)) + print('already current : {}'.format(duplicate)) + print('no matching asset: {}'.format(unresolved)) + print('rejected (blank) : {}'.format(rejected)) + if mismatches: + print() + print('filename/embedded MachineNo disagreements ({}):'.format( + len(mismatches))) + for filename_no, embedded_no in mismatches[:20]: + print(' file={:<10} embedded={}'.format(filename_no, embedded_no)) + print(' These are likely misfiled; review before trusting them.') + if not args.commit: + print() + print('DRY RUN - nothing written. Re-run with --commit to apply.') + + +if __name__ == '__main__': + main() diff --git a/shopdb/plugins/alembic_template.py b/shopdb/plugins/alembic_template.py index 213a15c..9f9caca 100644 --- a/shopdb/plugins/alembic_template.py +++ b/shopdb/plugins/alembic_template.py @@ -47,6 +47,7 @@ logger = logging.getLogger('alembic.env.plugin') # Explicit table-ownership map. Adding tables to a plugin requires updating # this dict so the per-plugin migration knows which tables to include. PLUGIN_TABLE_OWNERS: dict[str, Iterable[str]] = { + 'backups': ('backuprevisions',), 'computers': ('computertypes', 'computers', 'computerinstalledapps', 'accessprotocols', 'computeraccess'), 'employees': ('directoryemployees',), diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index 5acbd2e..a00b064 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -46,6 +46,8 @@ CUTOVER_PLUGINS = ( # stamp '0001anchor'; measuringtools stamps its real baseline id. EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS} EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline' +# backups is also post-cutover: its 0001 really creates backuprevisions. +EXPECTED_HEAD_REVISION['backups'] = 'backups0001baseline' # geenforce adds the content-addressed blob store (manifestblobs) on top of its # baseline. EXPECTED_HEAD_REVISION['geenforce'] = 'geenforce0002blobs' diff --git a/tests/test_plugins/test_backups.py b/tests/test_plugins/test_backups.py new file mode 100644 index 0000000..55d1159 --- /dev/null +++ b/tests/test_plugins/test_backups.py @@ -0,0 +1,633 @@ +"""Tests for the backups plugin. + +Two layers: + + Pure codec tests exercise the NTLARS .reg <-> JSON round trip and the dialect + toggle. These need no app and are where the real risk lives: getting the + WOW6432Node dialect wrong is silent - a reg import of the NTLARS dialect on a + 64-bit box writes to a hive NTLARS never reads and still reports success. + + Collector tests cover the dedup rule that makes revision history usable. + GE-Enforce runs the collector every cycle across the fleet, so an unchanged + machine must produce 'noop', not another row. +""" + +import base64 +import json + +import pytest + +from plugins.backups.services import ntlars, registry + + +# A minimal but representative NTLARS export: the root key plus one subkey, +# both value types that actually occur in the 320 real backups (REG_SZ and +# REG_DWORD), written in the WOW6432Node dialect that scripted exports produce. +SAMPLEREG = ( + 'Windows Registry Editor Version 5.00\r\n' + '\r\n' + '; NTLARS DNC Registry Backup\r\n' + '\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC]' '\r\n' + '"COMPUTERNAME"="GGBX0NH3ESF"\r\n' + '\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\Btr]' '\r\n' + '"BTR Rate"="300"\r\n' + '"Auto Rewind"="YES"\r\n' + '"CmntLag"=dword:00000000\r\n' +) + + +def _asbytes(text): + """Encode as UTF-16LE with a BOM, which is what regedit and NTLARS emit.""" + return b'\xff\xfe' + text.encode('utf-16-le') + + +# ============================================================================= +# Codec: parsing +# ============================================================================= + +def test_parse_reads_utf16_with_bom(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + assert projection['schema'] == ntlars.SCHEMA + assert [k['path'] for k in projection['keys']] == ['', 'Btr'] + + +def test_parse_reads_utf8_without_bom(): + projection = ntlars.parse(SAMPLEREG.encode('utf-8')) + assert [k['path'] for k in projection['keys']] == ['', 'Btr'] + + +def test_parse_strips_the_root_so_storage_is_dialect_neutral(): + """Both dialects must parse to the same projection - that is the point.""" + wow = ntlars.parse(_asbytes(SAMPLEREG)) + plain = ntlars.parse(_asbytes( + SAMPLEREG.replace(r'SOFTWARE\WOW6432Node\GE', r'SOFTWARE\GE'))) + assert wow['keys'] == plain['keys'] + assert wow['sourcedialect'] == 'wow6432node' + assert plain['sourcedialect'] == 'ntlars' + + +def test_parse_records_value_types(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + btr = next(k for k in projection['keys'] if k['path'] == 'Btr') + assert btr['values']['BTR Rate'] == {'type': 'REG_SZ', 'data': '300'} + assert btr['values']['CmntLag'] == {'type': 'REG_DWORD', 'data': 0} + + +def test_parse_sorts_keys_and_values_for_stable_diffs(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + btr = next(k for k in projection['keys'] if k['path'] == 'Btr') + assert list(btr['values']) == sorted(btr['values']) + + +def test_parse_ignores_keys_outside_the_dnc_tree(): + """An unrelated hive in the same file must not be folded into the backup.""" + polluted = SAMPLEREG + ( + r'[HKEY_LOCAL_MACHINE\SOFTWARE\Some Other Vendor\Thing]' '\r\n' + '"Nope"="should not appear"\r\n' + ) + projection = ntlars.parse(_asbytes(polluted)) + allvalues = {name for k in projection['keys'] for name in k['values']} + assert 'Nope' not in allvalues + + +def test_parse_rejects_a_file_with_no_registry_header(): + with pytest.raises(ntlars.NtlarsParseError): + ntlars.parse(b'this is not a reg file') + + +def test_parse_rejects_a_reg_file_with_no_dnc_keys(): + other = ( + 'Windows Registry Editor Version 5.00\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\Unrelated]' '\r\n' + '"X"="1"\r\n' + ) + with pytest.raises(ntlars.NtlarsParseError): + ntlars.parse(_asbytes(other)) + + +# ============================================================================= +# Codec: rendering and the dialect toggle +# ============================================================================= + +def test_render_ntlars_dialect_omits_wow6432node(): + """This is the form the NTLARS Load... button expects.""" + projection = ntlars.parse(_asbytes(SAMPLEREG)) + text = ntlars.render(projection, dialect='ntlars').decode('utf-16-le') + assert r'SOFTWARE\GE Aircraft Engines\DNC' in text + assert 'WOW6432Node' not in text + + +def test_render_wow6432node_dialect_includes_it(): + """This is the form `reg import` needs on a 64-bit machine.""" + projection = ntlars.parse(_asbytes(SAMPLEREG)) + text = ntlars.render(projection, dialect='wow6432node').decode('utf-16-le') + assert r'SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC' in text + + +def test_render_emits_utf16le_with_bom_and_crlf(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + raw = ntlars.render(projection) + assert raw.startswith(b'\xff\xfe') + assert '\r\n' in raw.decode('utf-16-le') + + +def test_render_starts_with_the_registry_header(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + text = ntlars.render(projection).decode('utf-16-le') + assert text.lstrip('\ufeff').startswith(ntlars.HEADER) + + +def test_render_rejects_an_unknown_dialect(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + with pytest.raises(ValueError): + ntlars.render(projection, dialect='nonsense') + + +def test_render_includes_comments_when_given(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + text = ntlars.render(projection, comments=['machine 3204']).decode('utf-16-le') + assert '; machine 3204' in text + + +@pytest.mark.parametrize('dialect', ['ntlars', 'wow6432node']) +def test_roundtrip_is_lossless_through_both_dialects(dialect): + first = ntlars.parse(_asbytes(SAMPLEREG)) + second = ntlars.parse(ntlars.render(first, dialect=dialect)) + assert first['keys'] == second['keys'] + + +def test_dword_survives_the_roundtrip_as_a_dword(): + """A REG_DWORD silently becoming REG_SZ would restore a broken config.""" + first = ntlars.parse(_asbytes(SAMPLEREG)) + second = ntlars.parse(ntlars.render(first)) + btr = next(k for k in second['keys'] if k['path'] == 'Btr') + assert btr['values']['CmntLag']['type'] == 'REG_DWORD' + + +# ============================================================================= +# Hashing / dedup key +# ============================================================================= + +def test_canonicalhash_is_stable_across_key_order(): + a = {'keys': [{'path': '', 'values': {'A': {'type': 'REG_SZ', 'data': '1'}}}]} + b = json.loads(json.dumps(a)) + assert registry.canonicalhash(a) == registry.canonicalhash(b) + + +def test_canonicalhash_changes_when_a_value_changes(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + before = registry.canonicalhash(projection) + changed = ntlars.parse(_asbytes(SAMPLEREG.replace('"300"', '"600"'))) + assert registry.canonicalhash(changed) != before + + +# ============================================================================= +# Kind registry +# ============================================================================= + +def test_ntlars_kind_stores_in_shopdb_and_offers_both_dialects(): + kind = registry.getkind('ntlars') + assert kind.storagebackend == 'shopdb' + assert {f['id'] for f in kind.formats()} == {'ntlars', 'wow6432node'} + + +def test_ntlars_default_format_is_the_load_button_dialect(): + """First format wins when the caller does not choose, so order matters.""" + assert registry.getkind('ntlars').formats()[0]['id'] == 'ntlars' + + +def test_partmarker_kind_is_share_backed_and_not_renderable(): + kind = registry.getkind('partmarker') + assert kind.storagebackend == 'share' + assert kind.formats() == [] + assert kind.parse(b'anything') is None + + +def test_partmarker_sharedir_is_under_the_sfld_backups_root(): + path = registry.getkind('partmarker').sharedir('lathe', '3204') + assert path.startswith(registry.DEFAULTSHAREROOT) + assert path.endswith(r'lathe\3204\partmarker') + + +def test_getkind_is_case_insensitive_and_returns_none_when_unknown(): + assert registry.getkind('NTLARS') is not None + assert registry.getkind('nosuchkind') is None + + +# ============================================================================= +# Blank-config guard +# ============================================================================= + +BLANKREG = ( + 'Windows Registry Editor Version 5.00\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\General]' '\r\n' + '"Cnc"=""\r\n' + '"MachineNo"=""\r\n' + '"HostType"=""\r\n' +) + +CONFIGUREDREG = BLANKREG.replace('"MachineNo"=""', '"MachineNo"="3204"') + + +def test_kind_rejects_an_unconfigured_ntlars_install(): + """A freshly imaged PC must not overwrite good history with a blank config.""" + with pytest.raises(ValueError, match='MachineNo'): + registry.getkind('ntlars').parse(_asbytes(BLANKREG)) + + +def test_kind_accepts_a_configured_install(): + projection = registry.getkind('ntlars').parse(_asbytes(CONFIGUREDREG)) + assert registry.NtlarsKind.embeddedmachineno(projection) == '3204' + + +def test_embeddedmachineno_is_empty_when_general_is_absent(): + projection = ntlars.parse(_asbytes(SAMPLEREG)) + assert registry.NtlarsKind.embeddedmachineno(projection) == '' + + +# ============================================================================= +# Collector path (apply_collector_payload) - the dedup rule that makes revision +# history usable. Needs an app context for the DB. +# ============================================================================= + +from shopdb import create_app # noqa: E402 +from shopdb.extensions import db as _db # noqa: E402 +from shopdb.plugins import plugin_manager # noqa: E402 + + +@pytest.fixture(scope='module') +def bk_app(): + """Testing app with the backups plugin registered, plus one machine asset.""" + saved = (plugin_manager._app, plugin_manager._db, plugin_manager.registry, + plugin_manager.loader, plugin_manager.migration_manager, + plugin_manager._registered_prefixes) + application = create_app('testing') + + from plugins.backups.plugin import BackupsPlugin + plugin = BackupsPlugin() + pm = application.extensions['plugin_manager'] + if plugin.meta.api_prefix not in pm._registered_prefixes: + pm._register_plugin_components(plugin) + + with application.app_context(): + _db.create_all() + from shopdb.core.models import AssetType, AssetStatus, Asset + assettype = AssetType(assettype='machine', description='Machine') + status = AssetStatus(status='In Use', description='In use') + _db.session.add_all([assettype, status]) + _db.session.flush() + # assetnumber is the machine number the collector reports. + _db.session.add(Asset(assetnumber='3204', name='Machine 3204', + assettypeid=assettype.assettypeid, + statusid=status.statusid)) + _db.session.commit() + yield application + _db.session.remove() + _db.drop_all() + + (plugin_manager._app, plugin_manager._db, plugin_manager.registry, + plugin_manager.loader, plugin_manager.migration_manager, + plugin_manager._registered_prefixes) = saved + + +@pytest.fixture +def bk_plugin(bk_app): + from plugins.backups.plugin import BackupsPlugin + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + _db.session.query(BackupRevision).delete() + _db.session.commit() + yield BackupsPlugin() + + +def _payload(reg=CONFIGUREDREG, **over): + data = { + 'machinenumber': '3204', + 'backupkind': 'ntlars', + 'contentbase64': base64.b64encode(_asbytes(reg)).decode('ascii'), + 'sourcehostname': 'GGBX0NH3ESF', + } + data.update(over) + return data + + +def test_first_post_creates_a_revision(bk_app, bk_plugin): + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload(_payload()) + assert result['action'] == 'created' + assert result['backuprevisionid'] + + +def test_identical_repost_is_a_noop_not_a_new_revision(bk_app, bk_plugin): + """GE-Enforce reposts every cycle; unchanged machines must not add rows.""" + with bk_app.app_context(): + first = bk_plugin.apply_collector_payload(_payload()) + second = bk_plugin.apply_collector_payload(_payload()) + assert second['action'] == 'noop' + assert second['backuprevisionid'] == first['backuprevisionid'] + + +def test_changed_settings_create_a_second_revision(bk_app, bk_plugin): + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + changed = CONFIGUREDREG.replace('"Cnc"=""', '"Cnc"="OKUMA"') + result = bk_plugin.apply_collector_payload(_payload(reg=changed)) + assert result['action'] == 'created' + + +def test_the_same_config_in_the_other_dialect_is_a_noop(bk_app, bk_plugin): + """Dedup must be dialect-neutral or the collection route causes churn.""" + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + plain = CONFIGUREDREG.replace(r'SOFTWARE\WOW6432Node\GE', r'SOFTWARE\GE') + result = bk_plugin.apply_collector_payload(_payload(reg=plain)) + assert result['action'] == 'noop' + + +def test_blank_config_is_rejected_through_the_collector(bk_app, bk_plugin): + with bk_app.app_context(): + with pytest.raises(ValueError, match='MachineNo'): + bk_plugin.apply_collector_payload(_payload(reg=BLANKREG)) + + +def test_unresolvable_machine_number_raises_rather_than_dropping(bk_app, bk_plugin): + with bk_app.app_context(): + with pytest.raises(ValueError, match='resolve'): + bk_plugin.apply_collector_payload(_payload(machinenumber='9999')) + + +def test_unknown_kind_raises(bk_app, bk_plugin): + with bk_app.app_context(): + with pytest.raises(ValueError, match='unknown backupkind'): + bk_plugin.apply_collector_payload(_payload(backupkind='nope')) + + +def test_machinenumber_disagreement_is_warned_not_hidden(bk_app, bk_plugin): + """PC configured for another machine must surface, not file silently.""" + with bk_app.app_context(): + other = CONFIGUREDREG.replace('"MachineNo"="3204"', '"MachineNo"="7602"') + result = bk_plugin.apply_collector_payload(_payload(reg=other)) + assert result['action'] == 'created' + assert any('7602' in w for w in result['warnings']) + + +def test_revision_attaches_to_the_machine_not_the_reporting_pc(bk_app, bk_plugin): + """The whole point: history survives replacement of the controlling PC.""" + from shopdb.core.models import Asset + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + revision = _db.session.query(BackupRevision).first() + machine = _db.session.get(Asset, revision.assetid) + assert machine.assetnumber == '3204' + assert revision.sourcehostname == 'GGBX0NH3ESF' + + +def test_a_replacement_pc_continues_the_same_history(bk_app, bk_plugin): + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + changed = CONFIGUREDREG.replace('"Cnc"=""', '"Cnc"="OKUMA"') + bk_plugin.apply_collector_payload( + _payload(reg=changed, sourcehostname='NEWPC001')) + revisions = _db.session.query(BackupRevision).all() + assert len(revisions) == 2 + assert {r.assetid for r in revisions} == {revisions[0].assetid} + assert {r.sourcehostname for r in revisions} == {'GGBX0NH3ESF', 'NEWPC001'} + + +def test_share_kind_requires_contenthash_and_sharepath(bk_app, bk_plugin): + with bk_app.app_context(): + with pytest.raises(ValueError, match='contenthash and sharepath'): + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker'}) + + +def test_share_kind_stores_the_pointer_without_a_payload(bk_app, bk_plugin): + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', + 'backupkind': 'partmarker', + 'contenthash': 'a' * 64, + 'sharepath': r'\\server\share\backups\lathe\3204\partmarker\x.tpm', + 'sourcefilename': 'x.tpm', + }) + assert result['action'] == 'created' + revision = _db.session.get(BackupRevision, result['backuprevisionid']) + assert revision.storagebackend == 'share' + assert revision.payload is None + assert revision.sourcefilename == 'x.tpm' + + +def test_collectedat_offset_is_converted_to_utc_not_truncated(bk_app, bk_plugin): + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload( + _payload(collectedat='2026-08-07T08:00:00-04:00')) + revision = _db.session.get(BackupRevision, result['backuprevisionid']) + assert revision.collectedat.hour == 12 + + +# ============================================================================= +# Retention +# ============================================================================= + +def _makerevisions(assetid, count): + from plugins.backups.models import BackupRevision + for i in range(count): + revision = BackupRevision( + assetid=assetid, backupkind='ntlars', storagebackend='shopdb', + contenthash='{:064d}'.format(i)) + revision.payload = {'schema': ntlars.SCHEMA, 'keys': []} + _db.session.add(revision) + _db.session.commit() + + +def test_prune_is_a_noop_when_both_limits_are_disabled(bk_app, bk_plugin): + from plugins.backups.services.retention import prune + from plugins.backups.models import BackupRevision + from shopdb.core.models import Asset + with bk_app.app_context(): + assetid = _db.session.query(Asset).first().assetid + _makerevisions(assetid, 10) + assert prune(assetid, 'ntlars', 0, 0) == 0 + assert _db.session.query(BackupRevision).count() == 10 + + +def test_prune_keeps_the_configured_count(bk_app, bk_plugin): + from plugins.backups.services.retention import prune + from plugins.backups.models import BackupRevision + from shopdb.core.models import Asset + with bk_app.app_context(): + assetid = _db.session.query(Asset).first().assetid + _makerevisions(assetid, 10) + removed = prune(assetid, 'ntlars', retentioncount=5) + _db.session.commit() + # 10 -> 5 newest, but the oldest is protected, so 6 survive. + assert removed == 4 + assert _db.session.query(BackupRevision).count() == 6 + + +def test_prune_never_deletes_the_newest_or_the_oldest(bk_app, bk_plugin): + """The newest is what a tech restores; the oldest is the baseline.""" + from plugins.backups.services.retention import prune + from plugins.backups.models import BackupRevision + from shopdb.core.models import Asset + with bk_app.app_context(): + assetid = _db.session.query(Asset).first().assetid + _makerevisions(assetid, 10) + ids = [r.backuprevisionid for r in + _db.session.query(BackupRevision) + .order_by(BackupRevision.backuprevisionid).all()] + prune(assetid, 'ntlars', retentioncount=1) + _db.session.commit() + surviving = {r.backuprevisionid for r in + _db.session.query(BackupRevision).all()} + assert ids[0] in surviving and ids[-1] in surviving + + +def test_prune_leaves_two_or_fewer_revisions_alone(bk_app, bk_plugin): + from plugins.backups.services.retention import prune + from shopdb.core.models import Asset + with bk_app.app_context(): + assetid = _db.session.query(Asset).first().assetid + _makerevisions(assetid, 2) + assert prune(assetid, 'ntlars', retentioncount=1) == 0 + + +# ============================================================================= +# DNC Info card +# ============================================================================= + +from plugins.backups.services import dncinfo # noqa: E402 + +DNCINFOREG = ( + 'Windows Registry Editor Version 5.00\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\General]' '\r\n' + '"MachineNo"="3204"\r\n' + '"Cnc"="OKUMA"\r\n' + '"NcIF"="EFOCAS"\r\n' + '"HostType"="WILM"\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\eFocas]' '\r\n' + '"IpAddr"="192.168.1.1"\r\n' + '"SocketNo"="8193"\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\Serial]' '\r\n' + '"Baud"="9600"\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\NTSHR]' '\r\n' + '"ShrHost"=""\r\n' + '"ShrFolder"=""\r\n\r\n' + r'[HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC\MARK]' '\r\n' + '"Baud"="9600"\r\n' + '"DncPatterns"="YES"\r\n' +) + + +def _headings(card): + return [f['label'] for f in card['fields'] if f.get('heading')] + + +def test_dncinfo_shows_efocas_and_serial(): + card = dncinfo.build(ntlars.parse(_asbytes(DNCINFOREG)), assetid=0) + assert any('eFocas' in h for h in _headings(card)) + assert any('Serial' in h for h in _headings(card)) + + +def test_dncinfo_hides_ntshr_when_it_has_no_content(): + """NTSHR is populated on only 18 of 147 machines; empty blocks are noise.""" + card = dncinfo.build(ntlars.parse(_asbytes(DNCINFOREG)), assetid=0) + assert not any('NTSHR' in h for h in _headings(card)) + + +def test_dncinfo_shows_ntshr_when_populated(): + populated = DNCINFOREG.replace('"ShrHost"=""', '"ShrHost"="WJFMS3"') + card = dncinfo.build(ntlars.parse(_asbytes(populated)), assetid=0) + assert any('NTSHR' in h for h in _headings(card)) + + +def test_dncinfo_hides_mark_for_a_machine_that_is_not_a_part_marker(): + """MARK carries serial defaults on 145 of 147 machines, so presence of the + key must NOT be what reveals the section.""" + card = dncinfo.build(ntlars.parse(_asbytes(DNCINFOREG)), assetid=0) + assert not any('MARK' in h for h in _headings(card)) + + +def test_dncinfo_shows_mark_when_the_asset_is_a_part_marker(monkeypatch): + monkeypatch.setattr(dncinfo, 'ispartmarker', lambda *a, **k: True) + card = dncinfo.build(ntlars.parse(_asbytes(DNCINFOREG)), assetid=0) + assert any('MARK' in h for h in _headings(card)) + + +def test_dncinfo_drops_empty_values_within_a_shown_section(): + populated = DNCINFOREG.replace('"ShrHost"=""', '"ShrHost"="WJFMS3"') + card = dncinfo.build(ntlars.parse(_asbytes(populated)), assetid=0) + labels = [f['label'] for f in card['fields'] if not f.get('heading')] + assert 'ShrHost' in labels + assert 'ShrFolder' not in labels + + +def test_dncinfo_is_empty_for_a_projection_with_nothing_interesting(): + card = dncinfo.build(ntlars.parse(_asbytes(CONFIGUREDREG)), assetid=0) + assert card['sectioncount'] == 0 + assert card['fields'] == [] + + +def test_ispartmarker_is_false_when_the_machines_plugin_is_absent(monkeypatch): + """Lean builds (ADR-014) may omit machines; the card must degrade, not error.""" + import builtins + real = builtins.__import__ + + def fake(name, *args, **kwargs): + if name.startswith('plugins.machines'): + raise ImportError('not installed') + return real(name, *args, **kwargs) + + monkeypatch.setattr(builtins, '__import__', fake) + assert dncinfo.ispartmarker(1) is False + + +def test_infopanel_is_kind_owned_so_a_successor_ships_its_own_card(): + """NTLARS/DNC is expected to give way to Shopfloor Connect; a new kind must + be able to add its card without touching the plugin or the endpoint.""" + ntlarskind = registry.getkind('ntlars') + panel = ntlarskind.infopanel() + assert panel['id'] == 'backups-dncinfo' + assert 'kind=ntlars' in panel['endpoint'] + # A kind with nothing to summarise contributes no panel at all. + assert registry.getkind('partmarker').infopanel() is None + + +def test_base_kind_buildinfo_is_an_empty_card(): + assert registry.BackupKind().buildinfo({}, 0) == {'fields': [], 'sectioncount': 0} + + +def test_dncinfo_general_section_leads_with_controller_identity(): + card = dncinfo.build(ntlars.parse(_asbytes(DNCINFOREG)), assetid=0) + assert _headings(card)[0] == 'General' + labels = [f['label'] for f in card['fields'] if not f.get('heading')] + assert 'Cnc' in labels and 'HostType' in labels + + +def test_dncinfo_general_omits_the_rest_of_the_key(): + """General carries 23 values; the card shows only the three that matter.""" + reg = DNCINFOREG.replace('"MachineNo"="3204"', + '"MachineNo"="3204"\r\n"Debug"="NO"\r\n"Site"="WJ"') + card = dncinfo.build(ntlars.parse(_asbytes(reg)), assetid=0) + labels = [f['label'] for f in card['fields'] if not f.get('heading')] + assert 'Debug' not in labels and 'Site' not in labels + + +def test_cnc_marker_reveals_the_mark_section_without_shopdb(): + """General\\Cnc='MARKER' is the one DNC-side part-marker signal, and works + before anyone sets the machine type in ShopDB.""" + reg = DNCINFOREG.replace('"Cnc"="OKUMA"', '"Cnc"="MARKER"') + card = dncinfo.build(ntlars.parse(_asbytes(reg)), assetid=0) + assert any('MARK' in h for h in _headings(card)) + + +def test_ordinary_controller_does_not_reveal_mark(): + reg = DNCINFOREG # fixture already carries Cnc=OKUMA + card = dncinfo.build(ntlars.parse(_asbytes(reg)), assetid=0) + assert not any('MARK' in h for h in _headings(card))