shopdb-flask/plugins/backups/models/backup.py

"""Backup revision model.

One row per DISTINCT configuration snapshot of an asset. The collector runs
every GE-Enforce cycle across the whole fleet, so the write path dedupes on
contenthash: a row appears only when a setting actually changed. That is what
turns a high-frequency collector into a readable revision history.

Two storage backends, chosen by the kind (see services/registry.py):

  'shopdb'  Parsed structured config lives in payloadjson. The original file
            is not kept because it re-renders exactly from the projection
            (NTLARS .reg is the motivating case), which also lets one stored
            revision render into more than one dialect on download.

  'share'   Opaque vendor formats that have no useful JSON representation
            (part marker files and similar). Bytes stay on the SFLD share and
            the row carries sharepath plus enough metadata to list, dedupe and
            link to them. ShopDB never needs to parse these.

contenthash is sha256 over the canonical form of whatever is authoritative for
the backend: the canonical JSON for 'shopdb', the raw file bytes for 'share'.
"""

import json
from datetime import datetime

from shopdb.api import db


class BackupRevision(db.Model):
    """A single point-in-time configuration snapshot of an asset."""

    __tablename__ = 'backuprevisions'

    backuprevisionid = db.Column(db.Integer, primary_key=True)

    # The asset the config BELONGS to, which is not always the asset it was
    # collected from. NTLARS settings live in the controlling PC's registry but
    # describe the machine, so the collector reports a machine number and the
    # kind resolves it to the machine's asset. sourcehostname records the PC it
    # actually came off.
    assetid = db.Column(
        db.Integer,
        db.ForeignKey('assets.assetid', ondelete='CASCADE'),
        nullable=False,
        index=True,
    )

    backupkind = db.Column(db.String(50), nullable=False, index=True)
    storagebackend = db.Column(db.String(20), nullable=False, default='shopdb')

    # sha256 of the canonical authoritative form. Dedup key together with
    # (assetid, backupkind).
    contenthash = db.Column(db.String(64), nullable=False, index=True)

    # Populated for storagebackend='shopdb' only. MEDIUMTEXT holding serialized
    # JSON rather than a native JSON column, matching geenforce.manifestjson.
    # Text also means the bytes come back exactly as written, so the canonical
    # key ordering the codec produces survives the round trip - a native JSON
    # column would renormalize it and make diffs between revisions unstable.
    payloadjson = db.Column(db.Text(length=16777215), nullable=True)

    # Populated for storagebackend='share' only. Full UNC path.
    sharepath = db.Column(db.String(500), nullable=True)

    # Original filename incl. extension. Vendor tools reject a renamed file, so
    # downloads hand back exactly this name.
    sourcefilename = db.Column(db.String(255), nullable=True)
    bytesize = db.Column(db.Integer, nullable=True)

    # Which PC produced it, and when it was read off that PC (not when ShopDB
    # stored it - a share-drop fallback can arrive much later).
    sourcehostname = db.Column(db.String(255), nullable=True)
    collectedat = db.Column(db.DateTime, nullable=True)

    createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)

    __table_args__ = (
        db.Index('ixbackuprevisionsassetkind', 'assetid', 'backupkind'),
    )

    @property
    def payload(self):
        """Decoded payloadjson, or None for opaque (share-backed) kinds."""
        if not self.payloadjson:
            return None
        try:
            return json.loads(self.payloadjson)
        except (ValueError, TypeError) as exc:
            # Do NOT return None here. A corrupt row would then look identical
            # to an opaque share-backed revision, and the download route would
            # fail somewhere further along with an unrelated error. Name the
            # actual problem and the row it is in.
            raise ValueError(
                'backuprevision {} has unreadable payloadjson: {}'.format(
                    self.backuprevisionid, exc))

    @payload.setter
    def payload(self, value):
        if value is None:
            self.payloadjson = None
        else:
            self.payloadjson = json.dumps(value, sort_keys=True,
                                          separators=(',', ':'))

    def to_dict(self, includepayload=False):
        data = {
            'backuprevisionid': self.backuprevisionid,
            'assetid': self.assetid,
            'backupkind': self.backupkind,
            'storagebackend': self.storagebackend,
            'contenthash': self.contenthash,
            'shorthash': (self.contenthash or '')[:12],
            'sharepath': self.sharepath,
            'sourcefilename': self.sourcefilename,
            'bytesize': self.bytesize,
            'sourcehostname': self.sourcehostname,
            'collectedat': self.collectedat.isoformat() if self.collectedat else None,
            'createdat': self.createdat.isoformat() if self.createdat else None,
        }
        if includepayload:
            data['payloadjson'] = self.payload
        return data