473 lines
18 KiB
Python
473 lines
18 KiB
Python
"""Versioned staging and retention for the endpoint agent binaries.
|
|
|
|
The site-agent packager ships a prebuilt shopdbagent.exe and an optional
|
|
shopdb-endpoint-tray.exe. Originally each was a single staged file under
|
|
instance/siteagent/; this module adds per-version staging so an operator can
|
|
drop in a new build, keep the last few, and roll a push out from a known
|
|
version.
|
|
|
|
Layout (all under the Flask instance directory):
|
|
|
|
siteagent/
|
|
shopdbagent.exe legacy flat path (pre-versioning)
|
|
shopdb-endpoint-tray.exe legacy flat path (pre-versioning)
|
|
agent/
|
|
manifest.json {current, versions:[{version,uploadedat,size}]}
|
|
1.6.3/shopdbagent.exe
|
|
1.6.2/shopdbagent.exe
|
|
tray/
|
|
manifest.json
|
|
1.0.0/shopdb-endpoint-tray.exe
|
|
|
|
Retention: the RETAIN most recently uploaded versions per binary are kept; the
|
|
current version is never pruned even if it falls outside that window. The legacy
|
|
flat file is treated as an unmanaged fallback: the packager still ships it when
|
|
no versioned build exists, so an install that predates versioning keeps working.
|
|
|
|
This module holds NO routes and NO permission checks - it is pure staging logic
|
|
so it can be unit-tested without a request context and reused by both the upload
|
|
routes and the packager. Callers own authorization.
|
|
"""
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import struct
|
|
from datetime import datetime
|
|
|
|
from flask import current_app
|
|
|
|
# How many versions to retain per binary. The current version is exempt from
|
|
# pruning, so the effective floor is max(RETAIN, 1-including-current).
|
|
RETAIN = 3
|
|
|
|
# Per-binary staging spec. 'exe' is the canonical filename shipped in the ZIP and
|
|
# stored inside each version directory; 'legacy' is the flat pre-versioning path
|
|
# under siteagent/ kept as a fallback source.
|
|
BINARY_SPECS = {
|
|
'agent': {'exe': 'shopdbagent.exe', 'legacy': 'shopdbagent.exe'},
|
|
'tray': {'exe': 'shopdb-endpoint-tray.exe', 'legacy': 'shopdb-endpoint-tray.exe'},
|
|
}
|
|
|
|
# A dotted numeric version, 1 to 4 parts (matches .NET assembly versions like
|
|
# 1.6.3 or 1.6.3.0). Deliberately strict so a typo or an arbitrary label cannot
|
|
# become a directory name.
|
|
_VERSION_RE = re.compile(r'^\d+(\.\d+){0,3}$')
|
|
|
|
# Windows PE magic. Uploaded agent binaries must be real PE executables; a quick
|
|
# header check rejects an operator who uploads the wrong file (a ZIP, a script)
|
|
# before it is ever staged or pushed.
|
|
_PE_MAGIC = b'MZ'
|
|
|
|
|
|
def is_valid_binary_key(binkey):
|
|
"""Whether binkey names a managed binary (agent or tray)."""
|
|
return binkey in BINARY_SPECS
|
|
|
|
|
|
def is_valid_version(version):
|
|
"""Whether version is a strict dotted-numeric string safe to use as a dir."""
|
|
return bool(version) and bool(_VERSION_RE.match(version))
|
|
|
|
|
|
def looks_like_pe(data):
|
|
"""Whether data begins with the Windows PE 'MZ' magic."""
|
|
return len(data) >= 2 and data[:2] == _PE_MAGIC
|
|
|
|
|
|
# RT_VERSION resource type id (the version resource Windows shows in file
|
|
# properties). We must read the version from THIS resource on the outer
|
|
# executable - not the first VS_FIXEDFILEINFO signature we can find in the bytes.
|
|
# A self-contained single-file .NET exe bundles dozens of framework assemblies,
|
|
# each carrying its own version block, so a naive byte scan returns a framework
|
|
# version (for example the .NET runtime's) instead of the app's. The only
|
|
# reliable value is the RT_VERSION resource reached by walking the PE resource
|
|
# directory, which belongs to the outer image alone.
|
|
_RT_VERSION = 16
|
|
|
|
# 0xFEEF04BD little-endian: the dwSignature that opens VS_FIXEDFILEINFO inside
|
|
# the RT_VERSION resource data.
|
|
_VS_FIXEDFILEINFO_SIGNATURE = b'\xbd\x04\xef\xfe'
|
|
|
|
|
|
def _rva_to_offset(data, rva, sections):
|
|
"""Map a resource RVA to a file offset using the section table, or None."""
|
|
for virtual_address, virtual_size, raw_offset, raw_size in sections:
|
|
if virtual_address <= rva < virtual_address + max(virtual_size, raw_size):
|
|
delta = rva - virtual_address
|
|
if delta < raw_size:
|
|
return raw_offset + delta
|
|
return None
|
|
|
|
|
|
def _first_resource_entry_offset(data, table_offset, resource_base, sections,
|
|
want_id):
|
|
"""Walk one IMAGE_RESOURCE_DIRECTORY level and return a child offset.
|
|
|
|
When want_id is not None, follow the entry whose id matches (used to select
|
|
RT_VERSION at the type level); otherwise follow the first entry (used to
|
|
descend name/language levels). Returns the absolute file offset of the child
|
|
directory or data entry, with the high bit stripped, or None.
|
|
"""
|
|
# IMAGE_RESOURCE_DIRECTORY: 12 fixed bytes, then NumberOfNamedEntries (2)
|
|
# and NumberOfIdEntries (2), then the entries.
|
|
named = struct.unpack_from('<H', data, table_offset + 12)[0]
|
|
id_count = struct.unpack_from('<H', data, table_offset + 14)[0]
|
|
entries_offset = table_offset + 16
|
|
for index in range(named + id_count):
|
|
entry_offset = entries_offset + index * 8
|
|
name_or_id, offset_to_data = struct.unpack_from('<II', data, entry_offset)
|
|
# Named entries carry the high bit in name_or_id; we only match numeric
|
|
# ids, so skip named entries when selecting by id.
|
|
is_named = bool(name_or_id & 0x80000000)
|
|
if want_id is not None:
|
|
if is_named or name_or_id != want_id:
|
|
continue
|
|
# Directory children have the high bit set on offset_to_data.
|
|
child = (offset_to_data & 0x7FFFFFFF) + resource_base
|
|
return child
|
|
return None
|
|
|
|
|
|
def read_pe_file_version(data):
|
|
"""Dotted four-part file version from the PE's RT_VERSION resource, or None.
|
|
|
|
Walks the PE resource directory to the RT_VERSION resource of the outer
|
|
executable and decodes its VS_FIXEDFILEINFO file version. Returns None when
|
|
the file has no version resource or the structures are malformed. Never
|
|
raises - callers treat None as "unknown, ask the operator".
|
|
"""
|
|
try:
|
|
if len(data) < 0x40 or data[:2] != _PE_MAGIC:
|
|
return None
|
|
pe_offset = struct.unpack_from('<I', data, 0x3C)[0]
|
|
if data[pe_offset:pe_offset + 4] != b'PE\x00\x00':
|
|
return None
|
|
|
|
coff = pe_offset + 4
|
|
number_of_sections = struct.unpack_from('<H', data, coff + 2)[0]
|
|
size_of_optional = struct.unpack_from('<H', data, coff + 16)[0]
|
|
optional = coff + 20
|
|
# Magic distinguishes PE32 (0x10b) from PE32+ (0x20b); the data-directory
|
|
# array sits at a different offset in each.
|
|
magic = struct.unpack_from('<H', data, optional)[0]
|
|
if magic == 0x20B:
|
|
data_dir = optional + 112
|
|
else:
|
|
data_dir = optional + 96
|
|
# Resource table is data directory index 2: its RVA then size.
|
|
resource_rva = struct.unpack_from('<I', data, data_dir + 2 * 8)[0]
|
|
if not resource_rva:
|
|
return None
|
|
|
|
section_table = optional + size_of_optional
|
|
sections = []
|
|
for index in range(number_of_sections):
|
|
base = section_table + index * 40
|
|
virtual_size = struct.unpack_from('<I', data, base + 8)[0]
|
|
virtual_address = struct.unpack_from('<I', data, base + 12)[0]
|
|
raw_size = struct.unpack_from('<I', data, base + 16)[0]
|
|
raw_offset = struct.unpack_from('<I', data, base + 20)[0]
|
|
sections.append((virtual_address, virtual_size, raw_offset, raw_size))
|
|
|
|
resource_base = _rva_to_offset(data, resource_rva, sections)
|
|
if resource_base is None:
|
|
return None
|
|
|
|
# Level 1: type -> pick RT_VERSION. Level 2: name/id -> first.
|
|
# Level 3: language -> first. Then a data entry pointing at the resource.
|
|
type_dir = _first_resource_entry_offset(
|
|
data, resource_base, resource_base, sections, _RT_VERSION)
|
|
if type_dir is None:
|
|
return None
|
|
name_dir = _first_resource_entry_offset(
|
|
data, type_dir, resource_base, sections, None)
|
|
if name_dir is None:
|
|
return None
|
|
lang_entry = _first_resource_entry_offset(
|
|
data, name_dir, resource_base, sections, None)
|
|
if lang_entry is None:
|
|
return None
|
|
|
|
# lang_entry is an IMAGE_RESOURCE_DATA_ENTRY: RVA (4), size (4), ...
|
|
version_rva = struct.unpack_from('<I', data, lang_entry)[0]
|
|
version_offset = _rva_to_offset(data, version_rva, sections)
|
|
if version_offset is None:
|
|
return None
|
|
|
|
# Within the version resource, find VS_FIXEDFILEINFO by its signature.
|
|
signature = data.find(_VS_FIXEDFILEINFO_SIGNATURE, version_offset,
|
|
version_offset + 512)
|
|
if signature == -1:
|
|
return None
|
|
base = signature + 4 + 4 # skip signature and dwStrucVersion
|
|
version_ms, version_ls = struct.unpack_from('<II', data, base)
|
|
parts = (
|
|
(version_ms >> 16) & 0xFFFF,
|
|
version_ms & 0xFFFF,
|
|
(version_ls >> 16) & 0xFFFF,
|
|
version_ls & 0xFFFF,
|
|
)
|
|
if not any(parts):
|
|
return None
|
|
return '.'.join(str(part) for part in parts)
|
|
except (struct.error, TypeError, IndexError):
|
|
return None
|
|
|
|
|
|
def read_pe_file_version_from_path(path):
|
|
"""Read a PE file's version from disk, or None on any failure."""
|
|
try:
|
|
with open(path, 'rb') as handle:
|
|
return read_pe_file_version(handle.read())
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def _siteagent_dir():
|
|
return os.path.join(current_app.instance_path, 'siteagent')
|
|
|
|
|
|
def _binary_dir(binkey):
|
|
return os.path.join(_siteagent_dir(), binkey)
|
|
|
|
|
|
def _manifest_path(binkey):
|
|
return os.path.join(_binary_dir(binkey), 'manifest.json')
|
|
|
|
|
|
def _legacy_path(binkey):
|
|
return os.path.join(_siteagent_dir(), BINARY_SPECS[binkey]['legacy'])
|
|
|
|
|
|
def _version_exe_path(binkey, version):
|
|
return os.path.join(_binary_dir(binkey), version, BINARY_SPECS[binkey]['exe'])
|
|
|
|
|
|
def _read_manifest(binkey):
|
|
"""Return the manifest dict, or an empty skeleton when none is written yet."""
|
|
path = _manifest_path(binkey)
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
data = json.load(handle)
|
|
except (OSError, ValueError):
|
|
return {'current': None, 'versions': []}
|
|
if not isinstance(data, dict):
|
|
return {'current': None, 'versions': []}
|
|
data.setdefault('current', None)
|
|
data.setdefault('versions', [])
|
|
return data
|
|
|
|
|
|
def _write_manifest(binkey, manifest):
|
|
os.makedirs(_binary_dir(binkey), exist_ok=True)
|
|
with open(_manifest_path(binkey), 'w', encoding='utf-8') as handle:
|
|
json.dump(manifest, handle, indent=2)
|
|
handle.write('\n')
|
|
|
|
|
|
def _version_sort_key(version):
|
|
"""Parse a dotted-numeric version into a tuple of ints for ordering.
|
|
|
|
Pads to four parts so 1.6 and 1.6.0.0 compare equal-ish and shorter
|
|
versions do not sort above longer ones with the same prefix. A part that
|
|
is not numeric (should not happen after validation) sorts as 0.
|
|
"""
|
|
parts = []
|
|
for chunk in str(version).split('.'):
|
|
try:
|
|
parts.append(int(chunk))
|
|
except ValueError:
|
|
parts.append(0)
|
|
while len(parts) < 4:
|
|
parts.append(0)
|
|
return tuple(parts[:4])
|
|
|
|
|
|
def list_versions(binkey):
|
|
"""Staged versions for a binary, highest version number first.
|
|
|
|
Each item is {version, uploadedat, size, iscurrent}. Reads the manifest as
|
|
the source of truth for currency; a version whose exe is missing on disk is
|
|
dropped so the caller never offers a broken push. Ordered by version number
|
|
(descending) so a rollback dropdown reads newest-build-first regardless of
|
|
the order versions were uploaded.
|
|
"""
|
|
manifest = _read_manifest(binkey)
|
|
current = manifest.get('current')
|
|
out = []
|
|
for entry in manifest.get('versions', []):
|
|
version = entry.get('version')
|
|
if not version or not os.path.isfile(_version_exe_path(binkey, version)):
|
|
continue
|
|
out.append({
|
|
'version': version,
|
|
'uploadedat': entry.get('uploadedat'),
|
|
'size': entry.get('size'),
|
|
'iscurrent': version == current,
|
|
})
|
|
out.sort(key=lambda item: _version_sort_key(item['version']), reverse=True)
|
|
return out
|
|
|
|
|
|
def current_version(binkey):
|
|
"""The current version string, or None when only a legacy flat file exists."""
|
|
manifest = _read_manifest(binkey)
|
|
current = manifest.get('current')
|
|
if current and os.path.isfile(_version_exe_path(binkey, current)):
|
|
return current
|
|
return None
|
|
|
|
|
|
def current_path(binkey):
|
|
"""Absolute path to the exe the packager should ship, or None.
|
|
|
|
Prefers the current managed version; falls back to the legacy flat file so a
|
|
site that staged a binary before versioning still packages it.
|
|
"""
|
|
version = current_version(binkey)
|
|
if version:
|
|
return _version_exe_path(binkey, version)
|
|
legacy = _legacy_path(binkey)
|
|
if os.path.isfile(legacy):
|
|
return legacy
|
|
return None
|
|
|
|
|
|
def legacy_status(binkey):
|
|
"""Describe an unmanaged legacy flat file, or None when there is none.
|
|
|
|
Returned only when a flat file exists and no managed version is current, so
|
|
the UI can show its detected version and offer one-click adoption without
|
|
the operator re-uploading. {version} is the PE file version if readable,
|
|
else None.
|
|
"""
|
|
if current_version(binkey):
|
|
return None
|
|
legacy = _legacy_path(binkey)
|
|
if not os.path.isfile(legacy):
|
|
return None
|
|
try:
|
|
size = os.path.getsize(legacy)
|
|
except OSError:
|
|
size = None
|
|
return {
|
|
'version': read_pe_file_version_from_path(legacy),
|
|
'size': size,
|
|
}
|
|
|
|
|
|
def adopt_legacy(binkey, version):
|
|
"""Copy the legacy flat file into the version store as a staged version.
|
|
|
|
Reads the flat file, validates it as a PE, and stores it under the given
|
|
version (making it current). Returns the fresh manifest view, or None when
|
|
there is no legacy file or it is not a PE. Caller supplies the version -
|
|
typically the detected PE version, or an operator override when the binary
|
|
carries no version resource.
|
|
"""
|
|
legacy = _legacy_path(binkey)
|
|
if not os.path.isfile(legacy):
|
|
return None
|
|
try:
|
|
with open(legacy, 'rb') as handle:
|
|
data = handle.read()
|
|
except OSError:
|
|
return None
|
|
if not looks_like_pe(data):
|
|
return None
|
|
return store_version(binkey, version, data)
|
|
|
|
|
|
def _prune(binkey, manifest):
|
|
"""Drop versions past the retention window, never removing the current one."""
|
|
current = manifest.get('current')
|
|
versions = manifest.get('versions', [])
|
|
keep = []
|
|
removed = 0
|
|
for index, entry in enumerate(versions):
|
|
version = entry.get('version')
|
|
# Keep the RETAIN newest (list is newest-first) plus the current always.
|
|
if index < RETAIN or version == current:
|
|
keep.append(entry)
|
|
continue
|
|
# Beyond the window and not current: remove its directory.
|
|
_remove_version_dir(binkey, version)
|
|
removed += 1
|
|
manifest['versions'] = keep
|
|
return removed
|
|
|
|
|
|
def _remove_version_dir(binkey, version):
|
|
directory = os.path.join(_binary_dir(binkey), version)
|
|
try:
|
|
exe = _version_exe_path(binkey, version)
|
|
if os.path.isfile(exe):
|
|
os.remove(exe)
|
|
if os.path.isdir(directory) and not os.listdir(directory):
|
|
os.rmdir(directory)
|
|
except OSError:
|
|
# Best effort: a locked or already-gone file must not fail the upload
|
|
# that triggered pruning. The manifest no longer references it either way.
|
|
pass
|
|
|
|
|
|
def store_version(binkey, version, data, make_current=True):
|
|
"""Stage an uploaded binary as a new version.
|
|
|
|
Writes data to siteagent/<binkey>/<version>/<exe>, records it newest-first in
|
|
the manifest (de-duping a re-upload of the same version in place), and prunes
|
|
older versions past RETAIN. When make_current is True the new version also
|
|
becomes the current staged build; pass False to stage a rollback/legacy build
|
|
that should appear in the version list without displacing the active current.
|
|
Returns the fresh manifest view {current, versions:[...]}. Caller must have
|
|
validated binkey, version, and that data is a PE binary.
|
|
"""
|
|
os.makedirs(os.path.join(_binary_dir(binkey), version), exist_ok=True)
|
|
with open(_version_exe_path(binkey, version), 'wb') as handle:
|
|
handle.write(data)
|
|
|
|
manifest = _read_manifest(binkey)
|
|
versions = [entry for entry in manifest.get('versions', [])
|
|
if entry.get('version') != version]
|
|
versions.insert(0, {
|
|
'version': version,
|
|
'uploadedat': datetime.utcnow().isoformat() + 'Z',
|
|
'size': len(data),
|
|
# The identity of the BYTES, not of the label. A version string is
|
|
# chosen by whoever uploads, and re-uploading the same string replaces
|
|
# the file in place - without this, a swap under a trusted label leaves
|
|
# no trace. It is also what an operator compares against the build they
|
|
# meant to ship, and what package time can check before handing an exe
|
|
# to the fleet.
|
|
'sha256': hashlib.sha256(data).hexdigest(),
|
|
})
|
|
manifest['versions'] = versions
|
|
if make_current or not manifest.get('current'):
|
|
manifest['current'] = version
|
|
_prune(binkey, manifest)
|
|
_write_manifest(binkey, manifest)
|
|
return {
|
|
'current': manifest['current'],
|
|
'versions': list_versions(binkey),
|
|
}
|
|
|
|
|
|
def set_current(binkey, version):
|
|
"""Point 'current' at an already-staged version. Returns True on success.
|
|
|
|
Used to roll back to a retained version. Refuses a version that is not
|
|
staged on disk so current never dangles.
|
|
"""
|
|
if not os.path.isfile(_version_exe_path(binkey, version)):
|
|
return False
|
|
manifest = _read_manifest(binkey)
|
|
if not any(entry.get('version') == version
|
|
for entry in manifest.get('versions', [])):
|
|
return False
|
|
manifest['current'] = version
|
|
_write_manifest(binkey, manifest)
|
|
return True
|