ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import the wilm agent store, WINC adoption work and batches CLI, with four fixes

Browse Source
This commit is contained in:
Cameron Proudlock
2026-09-03 08:16:38 -04:00
parent 4b21aa3887
commit b0bd73f62d
51 changed files with 5107 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/copilot-instructions.md vendored
View File

@@ -67,6 +67,11 @@ name. Spell out: `config context manager request response environment utilities`
`migrations/versions/`; each plugin owns its own chain under
`plugins/<name>/migrations/`. New schema = a new migration with an idempotent
guard and a real downgrade. Migrations must run clean on strict MySQL 8.
Any model column/table change ships WITH its migration in the SAME change, and
bringing up a new image MUST run `flask db upgrade` then
`flask plugin upgrade-all` before serving traffic. A model that references a
column its migration has not applied yet 500s every query on that table
(`1054 Unknown column`), so code and schema must never be deployed out of step.
- **Asset model is the platform contract.** Physical things are an `Asset` plus a
plugin subtype row linked by `assetid` (FK, `ON DELETE CASCADE`). Consumables
with quantities are standalone tables, not assets.

12
.github/workflows/publish-images.yml vendored
View File

@@ -76,8 +76,18 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Buildx
# Connect to a persistent, out-of-cluster BuildKit deployment instead of
# spinning up a builder against a local Docker daemon. The arc-runner-set
# pods have no Docker daemon (no /var/run/docker.sock) and cannot run a
# privileged DinD sidecar under cluster policy, so we point Buildx at a
# remote buildkitd Service. The endpoint is provided as a repo/org secret
# (e.g. tcp://buildkit.buildkit.svc.cluster.local:1234), so it is not
# hardcoded here. See deploy/ci/buildkit-remote.yaml for the deployment.
- name: Set up Buildx (remote BuildKit)
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: ${{ secrets.BUILDKIT_ENDPOINT }}
- name: Build and push channel image
uses: docker/build-push-action@v6

18
CONTRIBUTING.md
View File

@@ -24,6 +24,24 @@
- **Index names**: `idx_<table>_<column>`. Underscores allowed here only because indexes are infrastructure, not data.
- Good: `idx_machines_locationid`
### Migrations and deploy ordering
- **A model change ships WITH its migration in the same change.** Adding or
changing a column/table on a SQLAlchemy model without the matching Alembic
migration is a broken change, even if it passes tests against a freshly built
schema.
- **Code and schema must never deploy out of step.** A model that references a
column its migration has not applied yet makes MySQL raise
`1054 Unknown column`, which 500s every query on that table (this is not a
degraded read - the whole endpoint fails).
- **Bringing up a new image MUST run migrations before serving traffic**, in
order: `flask db upgrade` (core chain) then `flask plugin upgrade-all`
(per-plugin chains). A rebuilt image with new model code but an
un-migrated database will 500 until this runs.
- Core schema lives in `migrations/versions/`; each plugin owns its own chain
under `plugins/<name>/migrations/`. New schema = a new migration with an
idempotent guard and a real downgrade, clean on strict MySQL 8.
### Allowed acronyms (closed list, expand by PR)
- **Universal**: `id`, `url`, `api`, `http`, `https`, `json`, `jwt`, `sql`, `os`, `ip`, `dns`, `csv`, `pdf`, `cors`, `ttl`, `uuid`, `html`, `css`, `orm`

1
agent/ShopDbEndpointAgent.Core/Contracts.cs
View File

@@ -19,6 +19,7 @@ public sealed class EndpointReport
[JsonPropertyName("ipaddress")] public string? IpAddress { get; set; }
[JsonPropertyName("machinenumber")] public string? MachineNumber { get; set; }
[JsonPropertyName("agentversion")] public string? AgentVersion { get; set; }
[JsonPropertyName("trayversion")] public string? TrayVersion { get; set; }
[JsonPropertyName("lastboottime")] public DateTimeOffset? LastBootTime { get; set; }
[JsonPropertyName("uptimeseconds")] public long? UptimeSeconds { get; set; }
[JsonPropertyName("idleseconds")] public long? IdleSeconds { get; set; }

29
agent/ShopDbEndpointAgent.Core/EndpointCollector.cs
View File

@@ -37,6 +37,7 @@ public sealed class EndpointCollector : ICollector
{
Hostname = Environment.MachineName,
AgentVersion = agentVersion,
TrayVersion = GetTrayVersion(),
Adopt = adopt ? true : null,
LoggedInUser = GetLoggedInUser(),
SerialNumber = GetSerialNumber(),
@@ -65,6 +66,34 @@ public sealed class EndpointCollector : ICollector
return report;
}
// The interactive tray app ships and updates independently of this service,
// so the agent reports the tray's file version as its own field. Resolve the
// tray exe beside the agent first (the normal install layout), then the
// default install dir; degrade to null when it is absent or unreadable so a
// tray-less install simply omits the field rather than failing the report.
private const string TrayExeName = "shopdb-endpoint-tray.exe";
private const string DefaultInstallDir = @"C:\Program Files\ShopDB\EndpointAgent";
private string? GetTrayVersion()
{
try
{
var beside = Path.Combine(AppContext.BaseDirectory, TrayExeName);
var installed = Path.Combine(DefaultInstallDir, TrayExeName);
var trayExe = File.Exists(beside) ? beside
: File.Exists(installed) ? installed
: null;
if (trayExe is null)
return null;
return FileVersionInfo.GetVersionInfo(trayExe).FileVersion;
}
catch (Exception ex)
{
logger.LogDebug(ex, "could not resolve tray version");
return null;
}
}
// Combined watch list: interactive processes (by image name) and Windows
// services (by service key name). Both land in report.Services with a shared
// {name, app, kind, state, running, count} shape so the server/UI renders

5
agent/ShopDbEndpointAgent.Tray/ShopDbEndpointAgent.Tray.csproj
View File

@@ -8,6 +8,11 @@
<UseWindowsForms>true</UseWindowsForms>
<RootNamespace>ShopDbEndpointAgent.Tray</RootNamespace>
<AssemblyName>shopdb-endpoint-tray</AssemblyName>
<!-- 1.0.0: first explicitly stamped tray version. The tray and the agent
service version independently (they are separate exes and can be pushed
apart), so the tray carries its own <Version> and the agent reports it
as trayversion alongside its own agentversion (AGENT-CONTRACT.md 2). -->
<Version>1.0.0</Version>
<InvariantGlobalization>true</InvariantGlobalization>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<SelfContained>true</SelfContained>

123
deploy/ci/buildkit-remote.yaml Normal file
View File

@@ -0,0 +1,123 @@
# Persistent, non-privileged BuildKit for the publish-images workflow.
#
# WHY this exists: the arc-runner-set runner pods have the docker/buildx CLIs
# but no Docker daemon and cannot run a privileged DinD sidecar under cluster
# policy. So image builds are delegated to this standalone buildkitd Service.
# The workflow (.github/workflows/publish-images.yml) points Buildx at it with
# driver: remote
# endpoint: ${{ secrets.BUILDKIT_ENDPOINT }}
# where BUILDKIT_ENDPOINT is set to:
# tcp://buildkit.buildkit.svc.cluster.local:1234
#
# This runs rootless buildkitd, so it needs NO privileged: true. It only needs
# a few unconfined-ish knobs the rootless image documents. A persistent
# Deployment (not a per-job container) means the build cache in the PVC survives
# across runs, so the heavy frontend npm build is only paid in full once.
#
# Cameron: apply once with `kubectl apply -f deploy/ci/buildkit-remote.yaml`.
# This changes cluster infra only. It does NOT modify any application code or
# any branch of the repo.
---
apiVersion: v1
kind: Namespace
metadata:
name: buildkit
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: buildkit
namespace: buildkit
labels:
app: buildkit
spec:
replicas: 1
selector:
matchLabels:
app: buildkit
template:
metadata:
labels:
app: buildkit
annotations:
# Required by rootless buildkitd to unshare user/network namespaces
# without full privilege. These are the values from the upstream
# rootless example; they are far weaker than privileged: true.
container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
spec:
containers:
- name: buildkitd
image: moby/buildkit:v0.16.0-rootless
args:
- --addr
- unix:///run/user/1000/buildkit/buildkitd.sock
- --addr
- tcp://0.0.0.0:1234
# TLS is terminated in-cluster; the Service is only reachable from
# runner pods on the internal network. If you want mTLS between the
# runner and buildkitd, mount certs here and add --tlscacert etc,
# then set the endpoint secret to the tcp:// address with certs.
- --oci-worker-no-process-sandbox
readinessProbe:
exec:
command:
- buildctl
- debug
- workers
initialDelaySeconds: 5
periodSeconds: 30
livenessProbe:
exec:
command:
- buildctl
- debug
- workers
initialDelaySeconds: 5
periodSeconds: 30
securityContext:
# Rootless: no privilege escalation, runs as uid 1000. The seccomp
# and AppArmor unconfined settings below are what rootless needs in
# place of privileged: true.
runAsUser: 1000
runAsGroup: 1000
allowPrivilegeEscalation: false
seccompProfile:
type: Unconfined
ports:
- containerPort: 1234
name: buildkit
volumeMounts:
- name: buildkit-cache
mountPath: /home/user/.local/share/buildkit
volumes:
- name: buildkit-cache
persistentVolumeClaim:
claimName: buildkit-cache
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: buildkit-cache
namespace: buildkit
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
# Layer + build cache for the app image. Bump if the cache fills.
storage: 20Gi
---
apiVersion: v1
kind: Service
metadata:
name: buildkit
namespace: buildkit
spec:
selector:
app: buildkit
ports:
- name: buildkit
port: 1234
targetPort: 1234
# ClusterIP only: reachable from runner pods, not exposed outside the cluster.
type: ClusterIP

5
docker-compose.yml
View File

@@ -63,6 +63,11 @@ services:
ZABBIX_TOKEN: ${ZABBIX_TOKEN:-}
COLLECTOR_API_KEY: ${COLLECTOR_API_KEY:-}
COLLECTOR_API_KEY_ENDPOINTS: ${COLLECTOR_API_KEY_ENDPOINTS:-}
# This file is the source-build dev stack (wilm/dev-be); stable and edge
# ship through the pull/airgap compose files, not this one. So default the
# badge label to DEV here - a raw source build is by definition not a
# release - while still allowing an override from .env.
SHOPDB_ENV_LABEL: ${SHOPDB_ENV_LABEL:-DEV}
ports:
- "${API_PORT:-5001}:5001"
# Terminate TLS in gunicorn with the mounted self-signed cert so the browser

489
docs/openapi.json
View File

@@ -1973,6 +1973,194 @@
}
}
},
"/api/settings/asset-import/winc/preview": {
"post": {
"tags": [
"core-identity"
],
"summary": "Dry run a WINC machine-adoption import. Writes nothing; returns the plan, a summary, the needs-create groups an admin...",
"description": "Dry run a WINC machine-adoption import. Writes nothing; returns the plan, a summary, the needs-create groups an admin must acknowledge, and the planhash that authorises the matching commit\n\n**Auth:** jwt + require_role admin\n\n**Params:** multipart/form-data: files (one WINC/Maximo CSV export)\n\n**Example:**\n```\ncurl -X POST http://localhost:5001/api/settings/asset-import/winc/preview -H \"Authorization: Bearer $TOK\" -F files=@winc.csv\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
}
},
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "multipart/form-data: files (one WINC/Maximo CSV export)"
}
}
}
}
}
},
"/api/settings/asset-import/winc/commit": {
"post": {
"tags": [
"core-identity"
],
"summary": "Apply a previewed WINC plan in one transaction, journalled so it can be undone. planhash is required and must match a...",
"description": "Apply a previewed WINC plan in one transaction, journalled so it can be undone. planhash is required and must match a freshly rebuilt plan: without it the drift check compared a packet against the plan it was built from and could never fail, so an admin could approve one plan and apply another\n\n**Auth:** jwt + require_role admin\n\n**Params:** multipart/form-data: files (the same export); form: planhash (REQUIRED, from the preview), acknowledge (comma-separated needs-create group names)\n\n**Example:**\n```\ncurl -X POST http://localhost:5001/api/settings/asset-import/winc/commit -H \"Authorization: Bearer $TOK\" -F files=@winc.csv -F planhash=$HASH -F acknowledge=businessunit\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
}
},
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "multipart/form-data: files (the same export); form: planhash (REQUIRED, from the preview), acknowledge (comma-separated needs-create group names)"
}
}
}
}
}
},
"/api/settings/asset-import/winc/batches": {
"get": {
"tags": [
"core-identity"
],
"summary": "List applied and undone import batches, newest first, with kind, row count, who applied it and when",
"description": "List applied and undone import batches, newest first, with kind, row count, who applied it and when\n\n**Auth:** jwt + require_role admin\n\n**Params:** none\n\n**Example:**\n```\ncurl http://localhost:5001/api/settings/asset-import/winc/batches -H \"Authorization: Bearer $TOK\"\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
}
}
}
},
"/api/settings/asset-import/winc/undo": {
"post": {
"tags": [
"core-identity"
],
"summary": "Reverse an applied batch by replaying its journal backwards. Conservative: a cell is put back only if it still holds...",
"description": "Reverse an applied batch by replaying its journal backwards. Conservative: a cell is put back only if it still holds what the import wrote, and a created row is deleted only when nothing outside the batch references it\n\n**Auth:** jwt + require_role admin\n\n**Params:** body: batchid (required)\n\n**Example:**\n```\ncurl -X POST http://localhost:5001/api/settings/asset-import/winc/undo -H \"Authorization: Bearer $TOK\" -H 'Content-Type: application/json' -d '{\"batchid\":\"...\"}'\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
}
},
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "body: batchid (required)"
}
}
}
}
}
},
"/api/reports": {
"get": {
"tags": [
@@ -23905,6 +24093,307 @@
}
}
}
},
"/api/endpoints/siteagent/binaries/{binkey}": {
"get": {
"tags": [
"endpoints-siteagent"
],
"summary": "List the staged versions of one agent binary, newest first, with the current one marked",
"description": "List the staged versions of one agent binary, newest first, with the current one marked\n\n**Auth:** jwt + require_permission endpoints.agentpackage\n\n**Params:** path: binkey (agent|tray)\n\n**Example:**\n```\ncurl http://localhost:5001/api/endpoints/siteagent/binaries/agent -H \"Authorization: Bearer $TOK\"\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
},
"404": {
"description": "No such record."
}
},
"parameters": [
{
"name": "binkey",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
]
},
"post": {
"tags": [
"endpoints-siteagent"
],
"summary": "Stage an uploaded binary as a new version. This supplies the executable every future package installs and runs as...",
"description": "Stage an uploaded binary as a new version. This supplies the executable every future package installs and runs as SYSTEM on the fleet, so it is audit-logged with the stored file's sha256\n\n**Auth:** jwt + require_permission endpoints.agentpackage\n\n**Params:** path: binkey (agent|tray); multipart/form-data: file (a Windows PE, 64 MB cap), version (strict dotted-numeric), makecurrent (default true)\n\n**Example:**\n```\ncurl -X POST http://localhost:5001/api/endpoints/siteagent/binaries/agent -H \"Authorization: Bearer $TOK\" -F file=@shopdbagent.exe -F version=1.6.3\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
},
"404": {
"description": "No such record."
}
},
"parameters": [
{
"name": "binkey",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "path: binkey (agent|tray); multipart/form-data: file (a Windows PE, 64 MB cap), version (strict dotted-numeric), makecurrent (default true)"
}
}
}
}
}
},
"/api/endpoints/siteagent/binaries/{binkey}/detect": {
"post": {
"tags": [
"endpoints-siteagent"
],
"summary": "Read the version out of an uploaded PE's resource table so the upload form can prefill it. Stores nothing",
"description": "Read the version out of an uploaded PE's resource table so the upload form can prefill it. Stores nothing\n\n**Auth:** jwt + require_permission endpoints.agentpackage\n\n**Params:** path: binkey; multipart/form-data: file\n\n**Example:**\n```\ncurl -X POST http://localhost:5001/api/endpoints/siteagent/binaries/agent/detect -H \"Authorization: Bearer $TOK\" -F file=@shopdbagent.exe\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
},
"404": {
"description": "No such record."
}
},
"parameters": [
{
"name": "binkey",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "path: binkey; multipart/form-data: file"
}
}
}
}
}
},
"/api/endpoints/siteagent/binaries/{binkey}/adopt": {
"post": {
"tags": [
"endpoints-siteagent"
],
"summary": "Adopt a binary already staged on disk into the version manifest, for a build that predates the store",
"description": "Adopt a binary already staged on disk into the version manifest, for a build that predates the store\n\n**Auth:** jwt + require_permission endpoints.agentpackage\n\n**Params:** path: binkey; body: version\n\n**Example:**\n```\ncurl -X POST http://localhost:5001/api/endpoints/siteagent/binaries/agent/adopt -H \"Authorization: Bearer $TOK\" -H 'Content-Type: application/json' -d '{\"version\":\"1.6.2\"}'\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
},
"404": {
"description": "No such record."
}
},
"parameters": [
{
"name": "binkey",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "path: binkey; body: version"
}
}
}
}
}
},
"/api/endpoints/siteagent/binaries/{binkey}/current": {
"put": {
"tags": [
"endpoints-siteagent"
],
"summary": "Point the current staged build at another version, so a package rolls back to it. The retention sweep never prunes the...",
"description": "Point the current staged build at another version, so a package rolls back to it. The retention sweep never prunes the current one\n\n**Auth:** jwt + require_permission endpoints.agentpackage\n\n**Params:** path: binkey; body: version (must already be staged)\n\n**Example:**\n```\ncurl -X PUT http://localhost:5001/api/endpoints/siteagent/binaries/agent/current -H \"Authorization: Bearer $TOK\" -H 'Content-Type: application/json' -d '{\"version\":\"1.6.2\"}'\n```",
"security": [
{
"bearerAuth": []
}
],
"responses": {
"200": {
"description": "Success. Body is the success_response envelope: {status, data, meta}.",
"content": {
"application/json": {
"$ref": "#/components/schemas/SuccessEnvelope"
}
}
},
"default": {
"description": "Error. Body is the error envelope; the code and message are nested under data.error.",
"content": {
"application/json": {
"$ref": "#/components/schemas/ErrorEnvelope"
}
}
},
"401": {
"description": "Missing or invalid credentials."
},
"403": {
"description": "Authenticated, but not permitted."
},
"404": {
"description": "No such record."
}
},
"parameters": [
{
"name": "binkey",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true,
"description": "path: binkey; body: version (must already be staged)"
}
}
}
}
}
}
}
}

36
frontend/src/api/index.js
View File

@@ -1252,6 +1252,42 @@ export const settingsApi = {
return api.post('/settings/asset-import/import', form, {
headers: { 'Content-Type': 'multipart/form-data' }
})
},
wincPreview(file) {
// Read-only WINC (Maximo-style) machine-adoption analysis. Uploads one CSV
// and returns the approval packet (counts, needs-create business units,
// unresolved values); writes nothing.
const form = new FormData()
form.append('files', file)
return api.post('/settings/asset-import/winc/preview', form, {
headers: { 'Content-Type': 'multipart/form-data' }
})
},
wincCommit(file, acknowledge = [], planhash = '') {
// Apply a WINC plan reversibly. acknowledge is the list of needs-create
// group names (e.g. ['businessunit']) the admin confirmed. planhash is the
// one the PREVIEW returned - the server rejects a commit without it,
// because otherwise "approved" means only that these group names were
// ticked, and the plan applied could be a different one.
const form = new FormData()
form.append('files', file)
if (acknowledge.length) {
form.append('acknowledge', acknowledge.join(','))
}
if (planhash) {
form.append('planhash', planhash)
}
return api.post('/settings/asset-import/winc/commit', form, {
headers: { 'Content-Type': 'multipart/form-data' }
})
},
wincBatches() {
// Applied/undone WINC import batches, newest first, for the undo list.
return api.get('/settings/asset-import/winc/batches')
},
wincUndo(batchid) {
// Reverse an applied WINC import batch by id.
return api.post('/settings/asset-import/winc/undo', { batchid })
}
}

187
frontend/src/views/settings/ImportSiteAssets.spec.js Normal file
View File

@@ -0,0 +1,187 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { mount, flushPromises } from '@vue/test-utils'
// The WINC source is the one import path that is NOT a canonical-CSV adapter:
// it previews a machine-adoption plan, gates a commit behind an explicit
// business-unit acknowledgment, and exposes an undo list. These specs pin that
// gate and the preview/commit/undo wiring, mocking the settings API so no
// network or backend is involved.
vi.mock('@/api', () => ({
settingsApi: {
get: vi.fn().mockRejectedValue({ response: { status: 404 } }),
listImportTemplates: vi.fn().mockResolvedValue({ data: { data: { tables: [] } } }),
wincPreview: vi.fn(),
wincCommit: vi.fn(),
wincBatches: vi.fn().mockResolvedValue({ data: { data: { batches: [] } } }),
wincUndo: vi.fn().mockResolvedValue({ data: { data: { reverted: 3, conflicts: [] } } }),
},
computersApi: { types: { listAll: vi.fn().mockResolvedValue([]) } },
}))
vi.mock('@/utils/apiError', () => ({ apiError: (e) => String(e) }))
vi.mock('../../utils/apiError', () => ({ apiError: (e) => String(e) }))
import { settingsApi } from '@/api'
import ImportSiteAssets from './ImportSiteAssets.vue'
function previewReport(overrides = {}) {
return {
data: {
data: {
committed: false,
blocking: false,
rowcount: 1,
summary: { in_scope: 1, skipped_no_existing_machine: 0, ambiguous_match: 0 },
needscreate: { businessunit: { 'Large Parts': 1 } },
unresolved: {},
...overrides,
},
},
}
}
async function render() {
const wrapper = mount(ImportSiteAssets, {
global: {
stubs: { 'router-link': { template: '<a><slot /></a>' } },
},
})
await flushPromises()
return wrapper
}
function selectWinc(wrapper) {
// The WINC card is the fourth source-format button.
const card = wrapper.findAll('.source-card').find(c => c.text().includes('WINC'))
return card.trigger('click')
}
async function pickFile(wrapper) {
const input = wrapper.find('.winc-panel input[type="file"]')
const file = new File(['assetnum\nW0083A\n'], 'winc.csv', { type: 'text/csv' })
Object.defineProperty(input.element, 'files', { value: [file], configurable: true })
await input.trigger('change')
return file
}
describe('WINC source selection', () => {
beforeEach(() => {
settingsApi.wincPreview.mockReset()
settingsApi.wincCommit.mockReset()
})
it('offers WINC as a fourth source card', async () => {
const wrapper = await render()
const labels = wrapper.findAll('.source-card').map(c => c.text())
expect(labels.some(t => t.includes('WINC'))).toBe(true)
})
it('shows the WINC panel instead of the generic controls when selected', async () => {
const wrapper = await render()
await selectWinc(wrapper)
expect(wrapper.find('.winc-panel').exists()).toBe(true)
})
})
describe('WINC preview and commit gating', () => {
beforeEach(() => {
settingsApi.wincPreview.mockReset()
settingsApi.wincCommit.mockReset()
settingsApi.wincBatches.mockClear()
settingsApi.wincBatches.mockResolvedValue({ data: { data: { batches: [] } } })
})
it('previews the uploaded export and lists needs-create business units', async () => {
settingsApi.wincPreview.mockResolvedValue(previewReport())
const wrapper = await render()
await selectWinc(wrapper)
await pickFile(wrapper)
await wrapper.find('.winc-panel .btn-secondary').trigger('click')
await flushPromises()
expect(settingsApi.wincPreview).toHaveBeenCalledOnce()
expect(wrapper.find('.winc-needscreate').text()).toContain('Large Parts')
})
it('blocks commit until the business-unit creation is acknowledged', async () => {
settingsApi.wincPreview.mockResolvedValue(previewReport())
const wrapper = await render()
await selectWinc(wrapper)
await pickFile(wrapper)
await wrapper.find('.winc-panel .btn-secondary').trigger('click')
await flushPromises()
const commitBtn = wrapper.find('.winc-panel .btn-primary')
expect(commitBtn.attributes('disabled')).toBeDefined()
await wrapper.find('.winc-ack input[type="checkbox"]').setValue(true)
expect(commitBtn.attributes('disabled')).toBeUndefined()
})
it('commits with the businessunit acknowledgment and refreshes the batch list', async () => {
settingsApi.wincPreview.mockResolvedValue(previewReport())
settingsApi.wincCommit.mockResolvedValue({
data: { data: { committed: true, rowcount: 1, batchid: 'b-1', summary: { in_scope: 1, skipped_no_existing_machine: 0, ambiguous_match: 0 }, needscreate: {}, unresolved: {} } },
})
const wrapper = await render()
await selectWinc(wrapper)
await pickFile(wrapper)
await wrapper.find('.winc-panel .btn-secondary').trigger('click')
await flushPromises()
await wrapper.find('.winc-ack input[type="checkbox"]').setValue(true)
await wrapper.find('.winc-panel .btn-primary').trigger('click')
await flushPromises()
const [file, acknowledge] = settingsApi.wincCommit.mock.calls[0]
expect(file.name).toBe('winc.csv')
expect(acknowledge).toEqual(['businessunit'])
// Batches reloaded once on mount and once after commit.
expect(settingsApi.wincBatches).toHaveBeenCalledTimes(2)
})
it('does not require acknowledgment when there is nothing to create', async () => {
settingsApi.wincPreview.mockResolvedValue(previewReport({ needscreate: {} }))
const wrapper = await render()
await selectWinc(wrapper)
await pickFile(wrapper)
await wrapper.find('.winc-panel .btn-secondary').trigger('click')
await flushPromises()
expect(wrapper.find('.winc-ack').exists()).toBe(false)
expect(wrapper.find('.winc-panel .btn-primary').attributes('disabled')).toBeUndefined()
})
it('blocks commit when the plan is blocking', async () => {
settingsApi.wincPreview.mockResolvedValue(previewReport({
blocking: true, needscreate: {}, unresolved: { location: { 'WM-XYZ': 2 } },
}))
const wrapper = await render()
await selectWinc(wrapper)
await pickFile(wrapper)
await wrapper.find('.winc-panel .btn-secondary').trigger('click')
await flushPromises()
expect(wrapper.find('.winc-panel .btn-primary').attributes('disabled')).toBeDefined()
expect(wrapper.find('.problems').text()).toContain('WM-XYZ')
})
})
describe('WINC undo', () => {
it('undoes an applied batch and reloads the list', async () => {
settingsApi.wincBatches.mockResolvedValue({
data: { data: { batches: [
{ batchid: 'b-1', kind: 'winc-machine-adoption', rowcount: 5, createdat: '2026-01-01T00:00:00Z', createdby: 'admin', status: 'applied', undoneat: null },
] } },
})
const wrapper = await render()
await selectWinc(wrapper)
await flushPromises()
await wrapper.find('.winc-batches .btn-secondary').trigger('click')
await flushPromises()
expect(settingsApi.wincUndo).toHaveBeenCalledWith('b-1')
})
})

395
frontend/src/views/settings/ImportSiteAssets.vue
View File

@@ -112,6 +112,15 @@
:class="['source-card', { active: importSource === s.key }]"
@click="selectSource(s.key)"
>
<span class="source-card-head">
<svg class="source-card-icon" viewBox="0 0 24 24" aria-hidden="true">
<path :d="SOURCE_ICONS[s.icon]" fill="currentColor" />
</svg>
<span class="source-card-tag">{{ s.tag }}</span>
<svg v-if="importSource === s.key" class="source-card-check" viewBox="0 0 24 24" aria-hidden="true">
<path d="M9 16.2 4.8 12l-1.4 1.4L9 19 21 7l-1.4-1.4z" fill="currentColor" />
</svg>
</span>
<span class="source-card-title">{{ s.label }}</span>
<span class="source-card-desc">{{ s.description }}</span>
</button>
@@ -204,7 +213,166 @@
</p>
</div>
<div class="import-controls">
<div v-if="importSource === 'winc'" class="winc-panel">
<p class="muted">
A WINC (Maximo-style) asset export adopts existing machines: it never
creates an asset, only updates a machine that already exists, matched
on its workstation number. It sets the machine's name, location, and
business unit. Preview first - it writes nothing and reports what would
change, which business units it would create (you must confirm), and
any values it cannot resolve (which block the import). Every commit is
journalled, so it can be undone from the list below.
</p>
<div class="import-controls">
<input
ref="wincFileInput"
type="file"
accept=".csv"
@change="onWincFilePicked"
/>
<div class="actions">
<button
class="btn btn-secondary"
:disabled="!wincFile || wincPreviewing || wincCommitting"
@click="wincPreview"
>
{{ wincPreviewing ? 'Checking...' : 'Preview (dry run)' }}
</button>
<button
class="btn btn-primary"
:disabled="!wincCanCommit"
@click="wincCommit"
>
{{ wincCommitting ? 'Importing...' : 'Commit Import' }}
</button>
<button
v-if="wincFile"
class="btn btn-secondary btn-sm"
:disabled="wincPreviewing || wincCommitting"
@click="clearWincSelection"
>
Clear
</button>
</div>
</div>
<p v-if="wincFile" class="muted">Selected: {{ wincFile.name }}</p>
<p v-if="wincError" class="error">{{ wincError }}</p>
<div v-if="wincReport" class="report">
<p
:class="['report-banner', wincReport.committed ? 'ok' : (wincReport.blocking ? 'warn' : 'info')]"
>
<template v-if="wincReport.committed">
Imported: {{ wincReport.rowcount }} machine(s) updated. Undo it
below if needed.
</template>
<template v-else-if="wincReport.blocking">
{{ wincReport.rowcount }} row(s) in scope, but unresolved values
block the import - nothing was written. Resolve them and preview
again.
</template>
<template v-else>
Looks good: {{ wincReport.rowcount }} machine(s) would be updated.
Review below, then Commit Import.
</template>
</p>
<table class="report-table">
<tbody>
<tr>
<td>In scope (existing machines matched)</td>
<td>{{ wincReport.summary.in_scope }}</td>
</tr>
<tr>
<td>Skipped (no existing machine)</td>
<td>{{ wincReport.summary.skipped_no_existing_machine }}</td>
</tr>
<tr v-if="wincReport.summary.ambiguous_match">
<td>Ambiguous (skipped, not guessed)</td>
<td>{{ wincReport.summary.ambiguous_match }}</td>
</tr>
</tbody>
</table>
<div v-if="wincNeedsBusinessUnit" class="winc-needscreate">
<h4>New business units to be created</h4>
<p class="muted">
These business units are referenced by the export but do not exist
yet. Confirm they are correct - you can edit them later in
Settings. Nothing is created until you acknowledge and commit.
</p>
<ul>
<li v-for="(count, name) in wincReport.needscreate.businessunit" :key="name">
<strong>{{ name }}</strong> ({{ count }} machine(s))
</li>
</ul>
<label class="winc-ack" v-if="!wincReport.committed">
<input type="checkbox" v-model="wincAckBusinessUnit" />
I confirm these business units should be created.
</label>
</div>
<div v-if="wincReport.unresolved && Object.keys(wincReport.unresolved).length" class="problems">
<h4>Unresolved values (blocking)</h4>
<p class="muted">
These export values have no mapping yet, so the import will not
touch the affected rows. An administrator must add an alias for
each before this import can run.
</p>
<ul>
<li v-for="(items, group) in wincReport.unresolved" :key="group">
<strong>{{ group }}:</strong>
<span v-for="(count, value) in items" :key="value">
"{{ value }}" ({{ count }}){{ ' ' }}
</span>
</li>
</ul>
</div>
</div>
<div v-if="wincBatches.length" class="winc-batches">
<h4>Import history</h4>
<p class="muted">
Each committed WINC import can be undone. Undo reverses only cells
this import set that have not been changed since, and removes only
business units this import created.
</p>
<table class="report-table">
<thead>
<tr>
<th>When</th>
<th>By</th>
<th>Rows</th>
<th>Status</th>
<th></th>
</tr>
</thead>
<tbody>
<tr v-for="b in wincBatches" :key="b.batchid">
<td>{{ b.createdat }}</td>
<td>{{ b.createdby || '-' }}</td>
<td>{{ b.rowcount }}</td>
<td>{{ b.status }}</td>
<td>
<button
v-if="b.status === 'applied'"
class="btn btn-secondary btn-sm"
:disabled="wincUndoingId === b.batchid"
@click="undoWincBatch(b.batchid)"
>
{{ wincUndoingId === b.batchid ? 'Undoing...' : 'Undo' }}
</button>
<span v-else class="muted">undone</span>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<div v-if="importSource !== 'winc'" class="import-controls">
<input
ref="fileInput"
type="file"
@@ -448,11 +616,48 @@ const report = ref(null)
// Which format the selected files are in. 'native' means the uploads are
// already canonical per-table CSVs; 'azure' and 'thor' route through a
// server-side adapter that transforms the foreign export first.
// Each source carries a short tag (the kind of thing it writes) and an icon key
// so the picker reads at a glance. Icons are inline SVG paths (24x24 viewBox),
// kept ASCII-only per the house style - no emoji or icon-font glyphs.
const IMPORT_SOURCES = [
{ key: 'native', label: 'ShopDB native', description: 'The canonical per-table CSVs below (assets, computers, vendors, ...).' },
{ key: 'azure', label: 'Azure / Intune device group', description: 'A device-group member export from Entra. Creates PCs by hostname and stores the deviceId.' },
{ key: 'thor', label: 'Thor PC inventory', description: 'A Thor inventory export. Maps hostname, vendor, model, OS, user, and location.' }
{
key: 'native',
label: 'ShopDB native',
tag: 'All tables',
icon: 'table',
description: 'The canonical per-table CSVs below (assets, computers, vendors, ...).'
},
{
key: 'azure',
label: 'Azure / Intune device group',
tag: 'Computers',
icon: 'cloud',
description: 'A device-group member export from Entra. Creates PCs by hostname and stores the deviceId.'
},
{
key: 'thor',
label: 'Thor PC inventory',
tag: 'Computers',
icon: 'desktop',
description: 'A Thor inventory export. Maps hostname, vendor, model, OS, user, and location.'
},
{
key: 'winc',
label: 'Maximo Import',
tag: 'Machines',
icon: 'wrench',
description: 'A Maximo asset export (validated against the WINC / West Manufacturing site). Updates existing machines only; reversible, with an approval step.'
}
]
// Inline 24x24 SVG path data keyed by IMPORT_SOURCES[].icon. Single-color paths
// that inherit currentColor, so they tint with the active/inactive card state.
const SOURCE_ICONS = {
table: 'M3 5h18v14H3V5zm2 2v3h5V7H5zm7 0v3h7V7h-7zM5 12v5h5v-5H5zm7 0v5h7v-5h-7z',
cloud: 'M6.5 19a4.5 4.5 0 0 1-.5-8.97A6 6 0 0 1 17.66 9.1 4.5 4.5 0 0 1 17.5 19h-11z',
desktop: 'M3 4h18v12H3V4zm2 2v8h14V6H5zm4 12h6v2H9v-2z',
wrench: 'M21.7 6.3a5 5 0 0 1-6.4 6.4l-6.9 6.9a2 2 0 0 1-2.9-2.9l6.9-6.9a5 5 0 0 1 6.4-6.4l-2.8 2.8 1.4 3.5 3.5 1.4 2.8-2.8z'
}
const importSource = ref('native')
// Optional computer type stamped onto every computer an Azure import creates or
// updates, so a SHP group becomes Shopfloor PCs and a BUS group Inspection PCs.
@@ -463,6 +668,34 @@ const addingType = ref(false)
const computerTypes = ref([])
const azureDeviceCategory = ref('')
// WINC (Maximo-style) machine adoption is its own reversible path, not a
// canonical-CSV adapter: one uploaded export, an approval report, an explicit
// acknowledgment before any business unit is created, and an undo list.
const wincFile = ref(null)
const wincFileInput = ref(null)
const wincReport = ref(null)
const wincError = ref('')
const wincPreviewing = ref(false)
const wincCommitting = ref(false)
const wincAckBusinessUnit = ref(false)
const wincBatches = ref([])
const wincUndoingId = ref('')
// A WINC commit is allowed only after a clean, non-blocking preview of the
// current file, with every needs-create group acknowledged.
const wincNeedsBusinessUnit = computed(() =>
!!(wincReport.value && wincReport.value.needscreate
&& wincReport.value.needscreate.businessunit))
const wincCanCommit = computed(() => {
const wincResult = wincReport.value
if (!wincResult || wincCommitting.value || wincPreviewing.value) return false
if (wincResult.committed || wincResult.blocking) return false
if (wincResult.rowcount === 0) return false
if (wincNeedsBusinessUnit.value && !wincAckBusinessUnit.value) return false
return true
})
// Commit is allowed only after a clean preview of the CURRENT selection: a
// report with no problems and no request-level error that has not already been
// committed.
@@ -651,6 +884,82 @@ async function commit() {
}
}
function onWincFilePicked(event) {
wincFile.value = (event.target.files || [])[0] || null
// A new file invalidates the previous report and its acknowledgment.
wincReport.value = null
wincError.value = ''
wincAckBusinessUnit.value = false
}
function clearWincSelection() {
wincFile.value = null
wincReport.value = null
wincError.value = ''
wincAckBusinessUnit.value = false
if (wincFileInput.value) wincFileInput.value.value = ''
}
async function wincPreview() {
if (!wincFile.value) return
wincPreviewing.value = true
wincError.value = ''
wincAckBusinessUnit.value = false
try {
const response = await settingsApi.wincPreview(wincFile.value)
wincReport.value = response.data?.data || response.data || null
} catch (err) {
wincError.value = apiError(err, 'WINC preview failed.')
wincReport.value = null
} finally {
wincPreviewing.value = false
}
}
async function wincCommit() {
if (!wincCanCommit.value) return
wincCommitting.value = true
wincError.value = ''
try {
const acknowledge = wincNeedsBusinessUnit.value ? ['businessunit'] : []
// The hash from the preview the admin actually looked at. Without it the
// server refuses: a commit has to be able to prove it is applying the plan
// that was approved, not whatever the file says now.
const planhash = wincReport.value?.planhash || ''
const response = await settingsApi.wincCommit(wincFile.value, acknowledge,
planhash)
wincReport.value = response.data?.data || response.data || null
await loadWincBatches()
} catch (err) {
wincError.value = apiError(err, 'WINC import failed.')
} finally {
wincCommitting.value = false
}
}
async function loadWincBatches() {
try {
const response = await settingsApi.wincBatches()
const data = response.data?.data || response.data || {}
wincBatches.value = data.batches || []
} catch {
wincBatches.value = []
}
}
async function undoWincBatch(batchid) {
wincUndoingId.value = batchid
wincError.value = ''
try {
await settingsApi.wincUndo(batchid)
await loadWincBatches()
} catch (err) {
wincError.value = apiError(err, 'WINC undo failed.')
} finally {
wincUndoingId.value = ''
}
}
async function addComputerType() {
const name = customComputerType.value.trim()
if (!name) return
@@ -683,6 +992,7 @@ onMounted(async () => {
await loadLocation()
await loadTemplates()
await loadComputerTypes()
await loadWincBatches()
})
</script>
@@ -769,27 +1079,63 @@ onMounted(async () => {
.source-cards {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 10px;
margin: 4px 0;
gap: 12px;
margin: 6px 0;
}
.source-card {
display: flex;
flex-direction: column;
gap: 4px;
gap: 6px;
text-align: left;
padding: 12px 14px;
padding: 14px;
border: 1px solid var(--border);
border-radius: 8px;
border-radius: 10px;
background: var(--bg);
color: var(--text);
cursor: pointer;
position: relative;
transition: border-color 0.15s ease, box-shadow 0.15s ease, transform 0.15s ease, background 0.15s ease;
}
.source-card:hover {
border-color: var(--primary);
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.18);
transform: translateY(-1px);
}
.source-card.active {
border-color: var(--primary);
box-shadow: 0 0 0 1px var(--primary) inset;
box-shadow: 0 0 0 1px var(--primary) inset, 0 2px 10px rgba(0, 0, 0, 0.22);
background: color-mix(in srgb, var(--primary) 8%, var(--bg));
}
.source-card-head {
display: flex;
align-items: center;
gap: 8px;
}
.source-card-icon {
width: 22px;
height: 22px;
color: var(--text-light);
flex: none;
}
.source-card.active .source-card-icon {
color: var(--primary);
}
.source-card-tag {
font-size: 0.68rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.04em;
padding: 2px 8px;
border-radius: 999px;
background: color-mix(in srgb, var(--primary) 14%, transparent);
color: var(--primary);
}
.source-card-check {
width: 18px;
height: 18px;
color: var(--primary);
margin-left: auto;
flex: none;
}
.source-card-title {
font-weight: 600;
@@ -963,4 +1309,33 @@ onMounted(async () => {
margin: 4px 0;
font-size: 0.9rem;
}
.winc-panel {
margin-top: 12px;
}
.winc-needscreate {
margin-top: 12px;
border: 1px solid var(--warning, #d0a44a);
border-radius: 8px;
padding: 12px 14px;
}
.winc-needscreate h4 {
margin: 0 0 6px;
}
.winc-needscreate ul {
margin: 6px 0;
padding-left: 20px;
}
.winc-ack {
display: flex;
align-items: center;
gap: 8px;
margin-top: 8px;
font-size: 0.9rem;
}
.winc-batches {
margin-top: 16px;
}
.winc-batches h4 {
margin: 0 0 6px;
}
</style>

89
migrations/versions/7d37_import_batches_and_journal.py Normal file
View File

@@ -0,0 +1,89 @@
"""Add importbatches and importjournalentries (reversible bulk imports)
Two additive, platform-level tables that let any bulk-import writer record what
it changed so the change can be reversed later. Import-agnostic: an importer
writes primitive mutations (update a cell, insert a row) tied to a batch id, and
a generic undo replays them in reverse.
importbatches one row per approved write (identity: batchid, the same
value the approval packet carries).
importjournalentries one row per reversible primitive mutation, ordered by
sequence so undo can walk them in reverse.
Idempotent guards and a real downgrade throughout; strict MySQL 8 clean.
Revision ID: 7d37_import_batches_and_journal
Revises: 7d36_scriptlibrary_and_app_installer_digest
Create Date: 2026-09-03
"""
from alembic import op
import sqlalchemy as sa
revision = '7d37_import_batches_and_journal'
down_revision = '7d36_scriptlibrary_and_app_installer_digest'
branch_labels = None
depends_on = None
def upgrade():
bind = op.get_bind()
insp = sa.inspect(bind)
tables = insp.get_table_names()
if 'importbatches' not in tables:
op.create_table(
'importbatches',
sa.Column('batchid', sa.String(length=36), primary_key=True),
sa.Column('kind', sa.String(length=64), nullable=False),
sa.Column('planhash', sa.String(length=64), nullable=False),
sa.Column('rowcount', sa.Integer(), nullable=False,
server_default='0'),
sa.Column('createdat', sa.DateTime(), nullable=False),
sa.Column('createdby', sa.String(length=100), nullable=True),
sa.Column('status', sa.String(length=16), nullable=False,
server_default='applied'),
sa.Column('undoneat', sa.DateTime(), nullable=True),
)
if 'importjournalentries' not in tables:
op.create_table(
'importjournalentries',
sa.Column('entryid', sa.BigInteger(), primary_key=True,
autoincrement=True),
sa.Column('batchid', sa.String(length=36), nullable=False),
sa.Column('sequence', sa.Integer(), nullable=False),
sa.Column('operation', sa.String(length=16), nullable=False),
sa.Column('targettable', sa.String(length=64), nullable=False),
sa.Column('targetid', sa.BigInteger(), nullable=False),
sa.Column('columnname', sa.String(length=64), nullable=True),
sa.Column('oldvalue', sa.Text(), nullable=True),
sa.Column('newvalue', sa.Text(), nullable=True),
sa.Column('isreverted', sa.Boolean(), nullable=False,
server_default=sa.text('0')),
sa.ForeignKeyConstraint(
['batchid'], ['importbatches.batchid'],
name='fk_importjournalentries_batchid',
ondelete='CASCADE'),
)
op.create_index(
'idx_importjournalentries_batchid',
'importjournalentries', ['batchid'])
def downgrade():
bind = op.get_bind()
insp = sa.inspect(bind)
tables = insp.get_table_names()
if 'importjournalentries' in tables:
existing = [ix['name'] for ix in
insp.get_indexes('importjournalentries')]
if 'idx_importjournalentries_batchid' in existing:
op.drop_index('idx_importjournalentries_batchid',
table_name='importjournalentries')
op.drop_table('importjournalentries')
if 'importbatches' in tables:
op.drop_table('importbatches')

14
plugins/computers/frontend/views/PCDetail.vue
View File

@@ -9,9 +9,9 @@
look like a peer of printing a sticker. -->
<div class="header-actions">
<router-link :to="`/print/asset-label/computer/${$route.params.id}`" class="btn btn-secondary" target="_blank">Print Label</router-link>
<router-link :to="`/pcs/${$route.params.id}/edit`" class="btn btn-primary">Edit</router-link>
<router-link :to="`/pcs/${$route.params.id}/edit${fromQuery}`" class="btn btn-primary">Edit</router-link>
<button @click="confirmRetire" class="btn btn-danger">Retire</button>
<router-link to="/pcs" class="btn btn-secondary">Back to List</router-link>
<router-link :to="returnTo" class="btn btn-secondary">{{ returnLabel }}</router-link>
</div>
</div>
@@ -376,6 +376,14 @@ const toast = useToast()
const authStore = useAuthStore()
const { isEnabled } = useIdentifierFlags()
// Return-navigation origin. Fleet opens this page with ?from=fleet so Back,
// Edit, and Retire all lead back to Fleet; every other entry path falls back
// to the PCs list.
const fromFleet = computed(() => route.query.from === 'fleet')
const fromQuery = computed(() => (fromFleet.value ? '?from=fleet' : ''))
const returnTo = computed(() => (fromFleet.value ? '/endpoints' : '/pcs'))
const returnLabel = computed(() => (fromFleet.value ? 'Back to Fleet' : 'Back to List'))
const loading = ref(true)
const computer = ref(null)
// Liveness for the triage band. Null when the endpoints plugin is absent, or
@@ -643,7 +651,7 @@ async function confirmRetire() {
try {
await computersApi.delete(route.params.id)
toast.success(`Retired ${label}`)
router.push('/pcs')
router.push(returnTo.value)
} catch (error) {
console.error('Error retiring PC:', error)
toast.error(apiError(error, 'Failed to retire PC'))

9
plugins/computers/frontend/views/PCForm.vue
View File

@@ -353,7 +353,7 @@
<button type="submit" class="btn btn-primary" :disabled="saving">
{{ saving ? 'Saving...' : 'Save PC' }}
</button>
<router-link to="/pcs" class="btn btn-secondary">Cancel</router-link>
<router-link :to="returnTo" class="btn btn-secondary">Cancel</router-link>
</div>
</form>
</div>
@@ -385,6 +385,11 @@ const router = useRouter()
const isEdit = computed(() => !!route.params.id)
// Return-navigation origin, threaded through from Fleet via ?from=fleet so
// Cancel and a successful Save land back on Fleet; every other entry path
// falls back to the PCs list.
const returnTo = computed(() => (route.query.from === 'fleet' ? '/endpoints' : '/pcs'))
// Seeded asset-type id for computers (see /api/assets/types). Custom-field
// values are keyed by the underlying asset id, captured on load / create.
const COMPUTER_ASSETTYPEID = 2
@@ -685,7 +690,7 @@ async function savePC() {
}
}
router.push('/pcs')
router.push(returnTo.value)
} catch (err) {
console.error('Error saving PC:', err)
error.value = apiError(err, 'Failed to save PC')

2
plugins/endpoints/AGENT-CONTRACT.md
View File

@@ -32,6 +32,7 @@ arrays. `adopt` is first-contact only.
"pctype": "CNC Controller", // adoption subset
"machinenumber": "12345", // adoption subset; "" or "9999" == none
"agentversion": "1.0.0",
"trayversion": "1.0.0", // file version of the interactive tray exe, omitted if no tray
"lastboottime": "2025-01-14T06:12:00Z", // ISO-8601 UTC
"uptimeseconds": 86400,
"idleseconds": 120,
@@ -126,6 +127,7 @@ public sealed class EndpointReport
[JsonPropertyName("pctype")] public string? PcType { get; set; }
[JsonPropertyName("machinenumber")] public string? MachineNumber { get; set; }
[JsonPropertyName("agentversion")] public string? AgentVersion { get; set; }
[JsonPropertyName("trayversion")] public string? TrayVersion { get; set; }
[JsonPropertyName("lastboottime")] public DateTimeOffset? LastBootTime { get; set; }
[JsonPropertyName("uptimeseconds")] public long? UptimeSeconds { get; set; }
[JsonPropertyName("idleseconds")] public long? IdleSeconds { get; set; }

2
plugins/endpoints/api/__init__.py
View File

@@ -5,6 +5,8 @@ from .routes import endpoints_bp
from . import adoption # noqa: F401
# Attaches the site agent generator packaging routes to endpoints_bp.
from . import siteagent # noqa: F401
# Attaches the agent binary upload/retention routes to endpoints_bp.
from . import agentupload # noqa: F401
# Attaches the monitor catalog CRUD and resolve routes to endpoints_bp.
from . import monitors # noqa: F401

472
plugins/endpoints/api/agentstore.py Normal file
View File

@@ -0,0 +1,472 @@
"""Versioned staging and retention for the endpoint agent binaries.
The site-agent packager ships a prebuilt shopdbagent.exe and an optional
shopdb-endpoint-tray.exe. Originally each was a single staged file under
instance/siteagent/; this module adds per-version staging so an operator can
drop in a new build, keep the last few, and roll a push out from a known
version.
Layout (all under the Flask instance directory):
siteagent/
shopdbagent.exe legacy flat path (pre-versioning)
shopdb-endpoint-tray.exe legacy flat path (pre-versioning)
agent/
manifest.json {current, versions:[{version,uploadedat,size}]}
1.6.3/shopdbagent.exe
1.6.2/shopdbagent.exe
tray/
manifest.json
1.0.0/shopdb-endpoint-tray.exe
Retention: the RETAIN most recently uploaded versions per binary are kept; the
current version is never pruned even if it falls outside that window. The legacy
flat file is treated as an unmanaged fallback: the packager still ships it when
no versioned build exists, so an install that predates versioning keeps working.
This module holds NO routes and NO permission checks - it is pure staging logic
so it can be unit-tested without a request context and reused by both the upload
routes and the packager. Callers own authorization.
"""
import hashlib
import io
import json
import os
import re
import struct
from datetime import datetime
from flask import current_app
# How many versions to retain per binary. The current version is exempt from
# pruning, so the effective floor is max(RETAIN, 1-including-current).
RETAIN = 3
# Per-binary staging spec. 'exe' is the canonical filename shipped in the ZIP and
# stored inside each version directory; 'legacy' is the flat pre-versioning path
# under siteagent/ kept as a fallback source.
BINARY_SPECS = {
'agent': {'exe': 'shopdbagent.exe', 'legacy': 'shopdbagent.exe'},
'tray': {'exe': 'shopdb-endpoint-tray.exe', 'legacy': 'shopdb-endpoint-tray.exe'},
}
# A dotted numeric version, 1 to 4 parts (matches .NET assembly versions like
# 1.6.3 or 1.6.3.0). Deliberately strict so a typo or an arbitrary label cannot
# become a directory name.
_VERSION_RE = re.compile(r'^\d+(\.\d+){0,3}$')
# Windows PE magic. Uploaded agent binaries must be real PE executables; a quick
# header check rejects an operator who uploads the wrong file (a ZIP, a script)
# before it is ever staged or pushed.
_PE_MAGIC = b'MZ'
def is_valid_binary_key(binkey):
"""Whether binkey names a managed binary (agent or tray)."""
return binkey in BINARY_SPECS
def is_valid_version(version):
"""Whether version is a strict dotted-numeric string safe to use as a dir."""
return bool(version) and bool(_VERSION_RE.match(version))
def looks_like_pe(data):
"""Whether data begins with the Windows PE 'MZ' magic."""
return len(data) >= 2 and data[:2] == _PE_MAGIC
# RT_VERSION resource type id (the version resource Windows shows in file
# properties). We must read the version from THIS resource on the outer
# executable - not the first VS_FIXEDFILEINFO signature we can find in the bytes.
# A self-contained single-file .NET exe bundles dozens of framework assemblies,
# each carrying its own version block, so a naive byte scan returns a framework
# version (for example the .NET runtime's) instead of the app's. The only
# reliable value is the RT_VERSION resource reached by walking the PE resource
# directory, which belongs to the outer image alone.
_RT_VERSION = 16
# 0xFEEF04BD little-endian: the dwSignature that opens VS_FIXEDFILEINFO inside
# the RT_VERSION resource data.
_VS_FIXEDFILEINFO_SIGNATURE = b'\xbd\x04\xef\xfe'
def _rva_to_offset(data, rva, sections):
"""Map a resource RVA to a file offset using the section table, or None."""
for virtual_address, virtual_size, raw_offset, raw_size in sections:
if virtual_address <= rva < virtual_address + max(virtual_size, raw_size):
delta = rva - virtual_address
if delta < raw_size:
return raw_offset + delta
return None
def _first_resource_entry_offset(data, table_offset, resource_base, sections,
want_id):
"""Walk one IMAGE_RESOURCE_DIRECTORY level and return a child offset.
When want_id is not None, follow the entry whose id matches (used to select
RT_VERSION at the type level); otherwise follow the first entry (used to
descend name/language levels). Returns the absolute file offset of the child
directory or data entry, with the high bit stripped, or None.
"""
# IMAGE_RESOURCE_DIRECTORY: 12 fixed bytes, then NumberOfNamedEntries (2)
# and NumberOfIdEntries (2), then the entries.
named = struct.unpack_from('<H', data, table_offset + 12)[0]
id_count = struct.unpack_from('<H', data, table_offset + 14)[0]
entries_offset = table_offset + 16
for index in range(named + id_count):
entry_offset = entries_offset + index * 8
name_or_id, offset_to_data = struct.unpack_from('<II', data, entry_offset)
# Named entries carry the high bit in name_or_id; we only match numeric
# ids, so skip named entries when selecting by id.
is_named = bool(name_or_id & 0x80000000)
if want_id is not None:
if is_named or name_or_id != want_id:
continue
# Directory children have the high bit set on offset_to_data.
child = (offset_to_data & 0x7FFFFFFF) + resource_base
return child
return None
def read_pe_file_version(data):
"""Dotted four-part file version from the PE's RT_VERSION resource, or None.
Walks the PE resource directory to the RT_VERSION resource of the outer
executable and decodes its VS_FIXEDFILEINFO file version. Returns None when
the file has no version resource or the structures are malformed. Never
raises - callers treat None as "unknown, ask the operator".
"""
try:
if len(data) < 0x40 or data[:2] != _PE_MAGIC:
return None
pe_offset = struct.unpack_from('<I', data, 0x3C)[0]
if data[pe_offset:pe_offset + 4] != b'PE\x00\x00':
return None
coff = pe_offset + 4
number_of_sections = struct.unpack_from('<H', data, coff + 2)[0]
size_of_optional = struct.unpack_from('<H', data, coff + 16)[0]
optional = coff + 20
# Magic distinguishes PE32 (0x10b) from PE32+ (0x20b); the data-directory
# array sits at a different offset in each.
magic = struct.unpack_from('<H', data, optional)[0]
if magic == 0x20B:
data_dir = optional + 112
else:
data_dir = optional + 96
# Resource table is data directory index 2: its RVA then size.
resource_rva = struct.unpack_from('<I', data, data_dir + 2 * 8)[0]
if not resource_rva:
return None
section_table = optional + size_of_optional
sections = []
for index in range(number_of_sections):
base = section_table + index * 40
virtual_size = struct.unpack_from('<I', data, base + 8)[0]
virtual_address = struct.unpack_from('<I', data, base + 12)[0]
raw_size = struct.unpack_from('<I', data, base + 16)[0]
raw_offset = struct.unpack_from('<I', data, base + 20)[0]
sections.append((virtual_address, virtual_size, raw_offset, raw_size))
resource_base = _rva_to_offset(data, resource_rva, sections)
if resource_base is None:
return None
# Level 1: type -> pick RT_VERSION. Level 2: name/id -> first.
# Level 3: language -> first. Then a data entry pointing at the resource.
type_dir = _first_resource_entry_offset(
data, resource_base, resource_base, sections, _RT_VERSION)
if type_dir is None:
return None
name_dir = _first_resource_entry_offset(
data, type_dir, resource_base, sections, None)
if name_dir is None:
return None
lang_entry = _first_resource_entry_offset(
data, name_dir, resource_base, sections, None)
if lang_entry is None:
return None
# lang_entry is an IMAGE_RESOURCE_DATA_ENTRY: RVA (4), size (4), ...
version_rva = struct.unpack_from('<I', data, lang_entry)[0]
version_offset = _rva_to_offset(data, version_rva, sections)
if version_offset is None:
return None
# Within the version resource, find VS_FIXEDFILEINFO by its signature.
signature = data.find(_VS_FIXEDFILEINFO_SIGNATURE, version_offset,
version_offset + 512)
if signature == -1:
return None
base = signature + 4 + 4 # skip signature and dwStrucVersion
version_ms, version_ls = struct.unpack_from('<II', data, base)
parts = (
(version_ms >> 16) & 0xFFFF,
version_ms & 0xFFFF,
(version_ls >> 16) & 0xFFFF,
version_ls & 0xFFFF,
)
if not any(parts):
return None
return '.'.join(str(part) for part in parts)
except (struct.error, TypeError, IndexError):
return None
def read_pe_file_version_from_path(path):
"""Read a PE file's version from disk, or None on any failure."""
try:
with open(path, 'rb') as handle:
return read_pe_file_version(handle.read())
except OSError:
return None
def _siteagent_dir():
return os.path.join(current_app.instance_path, 'siteagent')
def _binary_dir(binkey):
return os.path.join(_siteagent_dir(), binkey)
def _manifest_path(binkey):
return os.path.join(_binary_dir(binkey), 'manifest.json')
def _legacy_path(binkey):
return os.path.join(_siteagent_dir(), BINARY_SPECS[binkey]['legacy'])
def _version_exe_path(binkey, version):
return os.path.join(_binary_dir(binkey), version, BINARY_SPECS[binkey]['exe'])
def _read_manifest(binkey):
"""Return the manifest dict, or an empty skeleton when none is written yet."""
path = _manifest_path(binkey)
try:
with open(path, 'r', encoding='utf-8') as handle:
data = json.load(handle)
except (OSError, ValueError):
return {'current': None, 'versions': []}
if not isinstance(data, dict):
return {'current': None, 'versions': []}
data.setdefault('current', None)
data.setdefault('versions', [])
return data
def _write_manifest(binkey, manifest):
os.makedirs(_binary_dir(binkey), exist_ok=True)
with open(_manifest_path(binkey), 'w', encoding='utf-8') as handle:
json.dump(manifest, handle, indent=2)
handle.write('\n')
def _version_sort_key(version):
"""Parse a dotted-numeric version into a tuple of ints for ordering.
Pads to four parts so 1.6 and 1.6.0.0 compare equal-ish and shorter
versions do not sort above longer ones with the same prefix. A part that
is not numeric (should not happen after validation) sorts as 0.
"""
parts = []
for chunk in str(version).split('.'):
try:
parts.append(int(chunk))
except ValueError:
parts.append(0)
while len(parts) < 4:
parts.append(0)
return tuple(parts[:4])
def list_versions(binkey):
"""Staged versions for a binary, highest version number first.
Each item is {version, uploadedat, size, iscurrent}. Reads the manifest as
the source of truth for currency; a version whose exe is missing on disk is
dropped so the caller never offers a broken push. Ordered by version number
(descending) so a rollback dropdown reads newest-build-first regardless of
the order versions were uploaded.
"""
manifest = _read_manifest(binkey)
current = manifest.get('current')
out = []
for entry in manifest.get('versions', []):
version = entry.get('version')
if not version or not os.path.isfile(_version_exe_path(binkey, version)):
continue
out.append({
'version': version,
'uploadedat': entry.get('uploadedat'),
'size': entry.get('size'),
'iscurrent': version == current,
})
out.sort(key=lambda item: _version_sort_key(item['version']), reverse=True)
return out
def current_version(binkey):
"""The current version string, or None when only a legacy flat file exists."""
manifest = _read_manifest(binkey)
current = manifest.get('current')
if current and os.path.isfile(_version_exe_path(binkey, current)):
return current
return None
def current_path(binkey):
"""Absolute path to the exe the packager should ship, or None.
Prefers the current managed version; falls back to the legacy flat file so a
site that staged a binary before versioning still packages it.
"""
version = current_version(binkey)
if version:
return _version_exe_path(binkey, version)
legacy = _legacy_path(binkey)
if os.path.isfile(legacy):
return legacy
return None
def legacy_status(binkey):
"""Describe an unmanaged legacy flat file, or None when there is none.
Returned only when a flat file exists and no managed version is current, so
the UI can show its detected version and offer one-click adoption without
the operator re-uploading. {version} is the PE file version if readable,
else None.
"""
if current_version(binkey):
return None
legacy = _legacy_path(binkey)
if not os.path.isfile(legacy):
return None
try:
size = os.path.getsize(legacy)
except OSError:
size = None
return {
'version': read_pe_file_version_from_path(legacy),
'size': size,
}
def adopt_legacy(binkey, version):
"""Copy the legacy flat file into the version store as a staged version.
Reads the flat file, validates it as a PE, and stores it under the given
version (making it current). Returns the fresh manifest view, or None when
there is no legacy file or it is not a PE. Caller supplies the version -
typically the detected PE version, or an operator override when the binary
carries no version resource.
"""
legacy = _legacy_path(binkey)
if not os.path.isfile(legacy):
return None
try:
with open(legacy, 'rb') as handle:
data = handle.read()
except OSError:
return None
if not looks_like_pe(data):
return None
return store_version(binkey, version, data)
def _prune(binkey, manifest):
"""Drop versions past the retention window, never removing the current one."""
current = manifest.get('current')
versions = manifest.get('versions', [])
keep = []
removed = 0
for index, entry in enumerate(versions):
version = entry.get('version')
# Keep the RETAIN newest (list is newest-first) plus the current always.
if index < RETAIN or version == current:
keep.append(entry)
continue
# Beyond the window and not current: remove its directory.
_remove_version_dir(binkey, version)
removed += 1
manifest['versions'] = keep
return removed
def _remove_version_dir(binkey, version):
directory = os.path.join(_binary_dir(binkey), version)
try:
exe = _version_exe_path(binkey, version)
if os.path.isfile(exe):
os.remove(exe)
if os.path.isdir(directory) and not os.listdir(directory):
os.rmdir(directory)
except OSError:
# Best effort: a locked or already-gone file must not fail the upload
# that triggered pruning. The manifest no longer references it either way.
pass
def store_version(binkey, version, data, make_current=True):
"""Stage an uploaded binary as a new version.
Writes data to siteagent/<binkey>/<version>/<exe>, records it newest-first in
the manifest (de-duping a re-upload of the same version in place), and prunes
older versions past RETAIN. When make_current is True the new version also
becomes the current staged build; pass False to stage a rollback/legacy build
that should appear in the version list without displacing the active current.
Returns the fresh manifest view {current, versions:[...]}. Caller must have
validated binkey, version, and that data is a PE binary.
"""
os.makedirs(os.path.join(_binary_dir(binkey), version), exist_ok=True)
with open(_version_exe_path(binkey, version), 'wb') as handle:
handle.write(data)
manifest = _read_manifest(binkey)
versions = [entry for entry in manifest.get('versions', [])
if entry.get('version') != version]
versions.insert(0, {
'version': version,
'uploadedat': datetime.utcnow().isoformat() + 'Z',
'size': len(data),
# The identity of the BYTES, not of the label. A version string is
# chosen by whoever uploads, and re-uploading the same string replaces
# the file in place - without this, a swap under a trusted label leaves
# no trace. It is also what an operator compares against the build they
# meant to ship, and what package time can check before handing an exe
# to the fleet.
'sha256': hashlib.sha256(data).hexdigest(),
})
manifest['versions'] = versions
if make_current or not manifest.get('current'):
manifest['current'] = version
_prune(binkey, manifest)
_write_manifest(binkey, manifest)
return {
'current': manifest['current'],
'versions': list_versions(binkey),
}
def set_current(binkey, version):
"""Point 'current' at an already-staged version. Returns True on success.
Used to roll back to a retained version. Refuses a version that is not
staged on disk so current never dangles.
"""
if not os.path.isfile(_version_exe_path(binkey, version)):
return False
manifest = _read_manifest(binkey)
if not any(entry.get('version') == version
for entry in manifest.get('versions', [])):
return False
manifest['current'] = version
_write_manifest(binkey, manifest)
return True

231
plugins/endpoints/api/agentupload.py Normal file
View File

@@ -0,0 +1,231 @@
"""Agent binary upload and retention routes.
Lets an operator drop in a new shopdbagent.exe or shopdb-endpoint-tray.exe from
the Endpoints page, keep the last few versions, and roll the current pointer
back to a retained build. The actual staging/retention logic lives in
agentstore.py; this module is only the HTTP surface and authorization.
Authorization boundary (deliberate):
- Uploading, listing, and selecting the current staged version is
endpoints.agentpackage - the same grant that already lets someone build and
download the site-agent ZIP. Staging a binary only changes what a FUTURE
download contains; it does not touch any running endpoint.
- Pushing that binary to live endpoints (agent replacement) is a separate,
heavier grant, endpoints.agentupdate, and is NOT wired here. It is a distinct
command family from application deploys (endpoints.deploy) so app-push rights
never imply SYSTEM-level agent replacement. That push pipe is a later slice.
Uploads are capped and PE-checked so a wrong or hostile file cannot be staged.
"""
from flask import request
from shopdb.api import (success_response, error_response, ErrorCodes, AuditLog, db,
require_permission)
from .routes import endpoints_bp
from . import agentstore
# Hard cap on an uploaded agent binary. The real agent is a few MB; this is a
# generous ceiling that still refuses an operator who points at a giant wrong
# file before it is buffered fully into memory-staged storage.
_MAX_UPLOAD_BYTES = 64 * 1024 * 1024
@endpoints_bp.route('/siteagent/binaries/<binkey>', methods=['GET'])
@require_permission('endpoints.agentpackage')
def list_agent_binaries(binkey):
"""List staged versions for a binary (agent or tray), newest first."""
if not agentstore.is_valid_binary_key(binkey):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'binkey must be one of: agent, tray',
http_code=400)
return success_response({
'binary': binkey,
'current': agentstore.current_version(binkey),
'versions': agentstore.list_versions(binkey),
})
@endpoints_bp.route('/siteagent/binaries/<binkey>', methods=['POST'])
@require_permission('endpoints.agentpackage')
def upload_agent_binary(binkey):
"""Stage an uploaded binary as a new version.
Expects multipart/form-data with a 'file' upload and a 'version' field. The
version must be a strict dotted-numeric string (for example 1.6.3) so it is
safe as a directory name and orderable. The upload must be a Windows PE
executable. Older versions past the retention window are pruned, but never
the current one. By default the upload becomes the current staged build; send
makecurrent=false to stage a rollback/legacy build that joins the version
list without displacing the active current.
"""
version = (request.form.get('version') or '').strip()
makecurrent = (request.form.get('makecurrent') or 'true').strip().lower()
makecurrent = makecurrent not in ('false', '0', 'no')
if not agentstore.is_valid_binary_key(binkey):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'binkey must be one of: agent, tray',
http_code=400)
if not agentstore.is_valid_version(version):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'version must be a dotted-numeric string such as 1.6.3',
http_code=400)
upload = request.files.get('file')
if upload is None or not upload.filename:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'a file upload is required',
http_code=400)
data = upload.read(_MAX_UPLOAD_BYTES + 1)
if len(data) > _MAX_UPLOAD_BYTES:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'uploaded binary exceeds the size limit',
http_code=400)
if not agentstore.looks_like_pe(data):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'uploaded file is not a Windows executable',
http_code=400)
result = agentstore.store_version(binkey, version, data,
make_current=makecurrent)
# This is the binary every future package installs and runs as SYSTEM on
# the fleet. Without a record, an incident has no answer to "who staged
# this, and when" - the manifest holds the file, not the person. The digest
# identifies the BYTES, because a version string is chosen by whoever
# uploads and re-uploading one replaces the file in place.
entry = next((v for v in result['versions']
if v.get('version') == version), {})
AuditLog.log(
'uploaded', 'AgentBinary', entityname='%s %s' % (binkey, version),
changes={'sha256': entry.get('sha256'), 'size': entry.get('size'),
'madecurrent': makecurrent})
db.session.commit()
return success_response({
'binary': binkey,
'current': result['current'],
'versions': result['versions'],
})
@endpoints_bp.route('/siteagent/binaries/<binkey>/detect', methods=['POST'])
@require_permission('endpoints.agentpackage')
def detect_agent_binary_version(binkey):
"""Read the PE file version of an uploaded EXE without staging it.
The Build Site Agent page calls this on file-select so the version field is
prefilled from the binary's own version resource - the operator does not
have to read the EXE properties by hand for a current-release upload.
Returns {version} which is null when the file carries no readable version.
"""
if not agentstore.is_valid_binary_key(binkey):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'binkey must be one of: agent, tray',
http_code=400)
upload = request.files.get('file')
if upload is None or not upload.filename:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'a file upload is required',
http_code=400)
data = upload.read(_MAX_UPLOAD_BYTES + 1)
if len(data) > _MAX_UPLOAD_BYTES:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'uploaded binary exceeds the size limit',
http_code=400)
if not agentstore.looks_like_pe(data):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'uploaded file is not a Windows executable',
http_code=400)
return success_response({
'binary': binkey,
'version': agentstore.read_pe_file_version(data),
})
@endpoints_bp.route('/siteagent/binaries/<binkey>/adopt', methods=['POST'])
@require_permission('endpoints.agentpackage')
def adopt_legacy_agent_binary(binkey):
"""Adopt the unmanaged legacy flat file into the version store.
The pre-versioning install dropped a single shopdbagent.exe (or tray exe)
straight under siteagent/. This copies that file in as a real staged version
so it shows a version and joins retention, without the operator having to
re-upload it. The version comes from the request when the binary carries no
readable PE version resource; otherwise the detected version is used.
"""
if not agentstore.is_valid_binary_key(binkey):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'binkey must be one of: agent, tray',
http_code=400)
status = agentstore.legacy_status(binkey)
if status is None:
return error_response(
ErrorCodes.NOT_FOUND,
'there is no unmanaged legacy binary to adopt',
http_code=404)
source = request.get_json(silent=True) or {}
version = (source.get('version') or status.get('version') or '').strip()
if not agentstore.is_valid_version(version):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'version must be a dotted-numeric string such as 1.6.3',
http_code=400)
result = agentstore.adopt_legacy(binkey, version)
if result is None:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'the legacy binary could not be adopted',
http_code=400)
return success_response({
'binary': binkey,
'current': result['current'],
'versions': result['versions'],
})
@endpoints_bp.route('/siteagent/binaries/<binkey>/current', methods=['PUT'])
@require_permission('endpoints.agentpackage')
def set_current_agent_binary(binkey):
"""Point 'current' at an already-staged version (roll back or forward)."""
if not agentstore.is_valid_binary_key(binkey):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'binkey must be one of: agent, tray',
http_code=400)
source = request.get_json(silent=True) or {}
version = (source.get('version') or '').strip()
if not agentstore.is_valid_version(version):
return error_response(
ErrorCodes.VALIDATION_ERROR,
'version must be a dotted-numeric string such as 1.6.3',
http_code=400)
if not agentstore.set_current(binkey, version):
return error_response(
ErrorCodes.NOT_FOUND,
'that version is not staged',
http_code=404)
return success_response({
'binary': binkey,
'current': agentstore.current_version(binkey),
'versions': agentstore.list_versions(binkey),
})

1
plugins/endpoints/api/routes.py
View File

@@ -444,6 +444,7 @@ def panel_status(asset_id: int):
field('CPU %', state.cpupercent),
field('Memory %', state.memorypercent),
field('Agent', state.agentversion, mono=True),
field('Tray', state.trayversion, mono=True),
field('Last reported',
state.lastreportedat.isoformat() + 'Z' if state.lastreportedat else None),
]

35
plugins/endpoints/api/siteagent.py
View File

@@ -35,6 +35,7 @@ from shopdb.api import (success_response, error_response, ErrorCodes,
require_permission)
from .routes import endpoints_bp
from . import agentstore
# Prebuilt, staged agent binaries. Built out of band and dropped under
# instance/siteagent/ (mirrors the adoption tool's STAGED_EXE_RELPATH model).
@@ -446,13 +447,23 @@ else {{
def _siteagent_status_payload():
"""Whether a prebuilt agent binary is staged for packaging."""
agent_staged = os.path.isfile(_staged_path(STAGED_AGENT_RELPATH))
tray_staged = os.path.isfile(_staged_path(STAGED_TRAY_RELPATH))
"""Whether a prebuilt agent binary is staged for packaging.
Reports both the legacy flat-file staging and the versioned store so the UI
can show current versions and an expandable retained-version history.
"""
agent_staged = agentstore.current_path('agent') is not None
tray_staged = agentstore.current_path('tray') is not None
return {
'agentstaged': agent_staged,
'traystaged': tray_staged,
'certavailable': _server_cert_path() is not None,
'agentcurrent': agentstore.current_version('agent'),
'traycurrent': agentstore.current_version('tray'),
'agentversions': agentstore.list_versions('agent'),
'trayversions': agentstore.list_versions('tray'),
'agentlegacy': agentstore.legacy_status('agent'),
'traylegacy': agentstore.legacy_status('tray'),
}
@@ -545,11 +556,11 @@ def package_siteagent():
if os.path.isfile(UNINSTALLER_PATH):
archive.write(UNINSTALLER_PATH, 'uninstall-shopdbagent.ps1')
agent = _staged_path(STAGED_AGENT_RELPATH)
if os.path.isfile(agent):
agent = agentstore.current_path('agent')
if agent and os.path.isfile(agent):
archive.write(agent, 'shopdbagent.exe')
tray = _staged_path(STAGED_TRAY_RELPATH)
if os.path.isfile(tray):
tray = agentstore.current_path('tray')
if tray and os.path.isfile(tray):
archive.write(tray, 'shopdb-endpoint-tray.exe')
if cert_pem:
archive.writestr('server.crt', cert_pem)
@@ -557,8 +568,16 @@ def package_siteagent():
archive.write(certpath, 'server.crt')
buffer.seek(0)
# Stamp the staged agent version into the download filename so a DT operator
# can see which agent build a package carries at a glance. This is cosmetic:
# install.ps1 finds the exe by its fixed name inside the ZIP, so the download
# name never affects the install. Falls back to the unversioned name when no
# managed version is current (only a legacy flat file is staged).
agentversion = agentstore.current_version('agent')
download_name = ('shopdb-endpoint-agent-{0}.zip'.format(agentversion)
if agentversion else 'shopdb-endpoint-agent.zip')
return send_file(
buffer,
mimetype='application/zip',
as_attachment=True,
download_name='shopdb-endpoint-agent.zip')
download_name=download_name)

413
plugins/endpoints/frontend/views/AgentKeySettings.vue
View File

@@ -106,8 +106,9 @@
</div>
<div class="card form-card">
<h4>How it works</h4>
<ul class="field-hint info-list">
<details class="collapsible">
<summary><h4>How it works</h4></summary>
<ul class="field-hint info-list">
<li>
<strong>Generate</strong> mints a fresh key and, on the first
generation, turns off the bootstrap <code>.env</code> fallback so
@@ -127,9 +128,11 @@
the kill switch.
</li>
</ul>
</details>
<h4>When to rotate</h4>
<ul class="field-hint info-list">
<details class="collapsible">
<summary><h4>When to rotate</h4></summary>
<ul class="field-hint info-list">
<li>
<strong>An installer or agent left the secure environment.</strong>
If a build carrying the key is copied off the shop floor - onto an
@@ -156,11 +159,12 @@
regenerate and redeploy to known-good endpoints.
</li>
</ul>
</details>
</div>
</section>
<!-- ====================================================================
Adoption tool (spike)
Adoption tool
================================================================= -->
<section class="settings-section">
<h3>Adoption tool</h3>
@@ -211,7 +215,7 @@
</section>
<!-- ====================================================================
Build site agent (placeholder)
Build site agent
================================================================= -->
<section class="settings-section">
<h3>Build site agent</h3>
@@ -227,10 +231,9 @@
</p>
<p v-if="!agentstaged" class="field-hint">
<strong>No agent binary is staged on this server.</strong> The package
will still build with config and installer, but you must drop
<code>shopdbagent.exe</code> beside them before deploying. Stage a
prebuilt agent under <code>instance/siteagent/</code> to have it ship
in the download.
will still build with config and installer, but no agent will ship
inside it. Upload a current release in the binary section below to
have it included in every download.
</p>
<label class="field-label" for="agent-baseurl">Server base URL</label>
@@ -303,6 +306,162 @@
</div>
<p v-if="agentmessage" class="field-hint">{{ agentmessage }}</p>
<p v-if="agenterror" class="field-error">{{ agenterror }}</p>
<!-- ================================================================
Agent binary versions: upload a new build, keep the last few,
and roll the current pointer back to a retained version. Staging
only changes what a FUTURE download contains; it does not touch
any running endpoint. Pushing to live endpoints is a separate,
heavier grant handled elsewhere. Collapsed by default - binary
management is infrequent next to building a package. -->
<details class="collapsible binary-manager">
<summary><h4>Upload New/Legacy Agents</h4></summary>
<div
v-for="binary in binaries"
:key="binary.key"
class="binary-block"
>
<h4>{{ binary.label }}</h4>
<p class="field-hint">
Upload {{ binary.name }} updated EXE. Uploading here will update the
binaries available for your "Build Site Agent" install package.
</p>
<p class="field-hint">
Current staged version:
<strong>{{ binary.current || 'none staged yet - upload a build below' }}</strong>
</p>
<!-- Current release upload. The version is auto-detected from the
EXE's PE version resource on select; the operator can override.
This upload becomes the current staged build. -->
<label class="field-label">
Upload {{ binary.name }} binary current release
</label>
<div class="binary-upload">
<input
class="form-control"
type="file"
accept=".exe"
@change="onBinaryFileChange(binary.key, $event)"
/>
<input
v-model="binary.version"
class="form-control"
type="text"
:placeholder="binary.detecting
? 'Reading version...'
: 'Version, auto-detected from EXE'"
/>
<button
class="btn btn-secondary"
type="button"
:disabled="binary.busy || !binary.version || !binary.file"
@click="uploadBinary(binary.key)"
>
{{ binary.busy ? 'Uploading...' : 'Upload current release' }}
</button>
</div>
<!-- Rollback / legacy path. Stages a build into the version list
without displacing the current release so it is available to
roll back to. WARNING copy per intent: the version must be
confirmed from the EXE properties before uploading. -->
<label class="field-label">
Upload legacy build for rollback
</label>
<p class="field-warning">
WARNING: confirm agent version before uploading from the EXE
properties and enter the version number.
</p>
<div class="binary-upload">
<input
class="form-control"
type="file"
accept=".exe"
@change="onLegacyFileChange(binary.key, $event)"
/>
<input
v-model="binary.legacyversion"
class="form-control"
type="text"
placeholder="Version, e.g. 1.5.0"
/>
<button
class="btn btn-secondary"
type="button"
:disabled="binary.busy || !binary.legacyversion || !binary.legacyfile"
@click="uploadLegacy(binary.key)"
>
{{ binary.busy ? 'Uploading...' : 'Upload legacy build' }}
</button>
</div>
<p v-if="binary.message" class="field-hint">{{ binary.message }}</p>
<p v-if="binary.error" class="field-error">{{ binary.error }}</p>
<!-- Version selector: pick the staged build shipped by the next
download. Ordered by version number (highest first). -->
<div v-if="binary.versions.length" class="binary-upload">
<label class="field-label" :for="`select-${binary.key}`">
Active version
</label>
<select
:id="`select-${binary.key}`"
class="form-control"
:value="binary.current || ''"
:disabled="binary.busy"
@change="setCurrentBinary(binary.key, $event.target.value)"
>
<option
v-for="entry in binary.versions"
:key="entry.version"
:value="entry.version"
>
{{ entry.version }}{{ entry.iscurrent ? ' (current)' : '' }}
</option>
</select>
</div>
<div v-if="binary.versions.length" class="previous-versions">
<button
class="btn-link"
type="button"
@click="binary.showprevious = !binary.showprevious"
>
{{ binary.showprevious ? 'Hide' : 'Show' }} staged versions
({{ binary.versions.length }})
</button>
<table v-if="binary.showprevious" class="versions-table">
<thead>
<tr>
<th>Version</th>
<th>Uploaded</th>
<th>Size</th>
<th></th>
</tr>
</thead>
<tbody>
<tr v-for="entry in binary.versions" :key="entry.version">
<td class="mono">{{ entry.version }}</td>
<td>{{ formatDate(entry.uploadedat) }}</td>
<td>{{ formatBytes(entry.size) }}</td>
<td>
<span v-if="entry.iscurrent" class="current-tag">current</span>
<button
v-else
class="btn-link"
type="button"
:disabled="binary.busy"
@click="setCurrentBinary(binary.key, entry.version)"
>
Make current
</button>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</details>
</div>
</section>
@@ -350,7 +509,7 @@ const copied = ref(false)
const message = ref('')
const error = ref('')
// Adoption tool (spike): build a one-shot, agentless enrollment download.
// Adoption tool: build a one-shot, agentless enrollment download.
const adoptbaseurl = ref('')
const adoptcollectorkey = ref('')
const adoptbusy = ref(false)
@@ -370,12 +529,45 @@ const agentbusy = ref(false)
const agentmessage = ref('')
const agenterror = ref('')
// Versioned agent/tray binaries. Each block owns its own upload form, retained
// version list, and busy/message state so the two rows act independently. The
// 'key' matches the backend binkey (agent | tray).
const binaries = ref([
{ key: 'agent', label: 'ShopDB Agent (shopdbagent.exe)',
name: 'ShopDB Agent',
current: null, versions: [], version: '', file: '', detecting: false,
legacyversion: '', legacyfile: '',
busy: false, message: '', error: '', showprevious: false },
{ key: 'tray', label: 'Tray Agent (shopdb-endpoint-tray.exe)',
name: 'Tray Agent',
current: null, versions: [], version: '', file: '', detecting: false,
legacyversion: '', legacyfile: '',
busy: false, message: '', error: '', showprevious: false },
])
function binaryFor(binkey) {
return binaries.value.find((entry) => entry.key === binkey)
}
function formatBytes(value) {
if (!value && value !== 0) return ''
if (value < 1024) return value + ' B'
if (value < 1024 * 1024) return (value / 1024).toFixed(1) + ' KB'
return (value / (1024 * 1024)).toFixed(1) + ' MB'
}
function formatDate(value) {
if (!value) return ''
const parsed = new Date(value)
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString()
}
function filenameFromResponse(response, fallback) {
const disposition = response.headers?.['content-disposition'] || ''
const match = /filename\*?=(?:UTF-8'')?"?([^";]+)"?/i.exec(disposition)
return match ? decodeURIComponent(match[1]) : fallback
}
function applyStatus(data) {
key.value = data.key || null
envkeyfallback.value = !!data.envkeyfallback
@@ -401,11 +593,20 @@ onMounted(async () => {
agentstaged.value = !!status.agentstaged
certavailable.value = !!status.certavailable
agentbundlecert.value = !!status.certavailable
applyBinaryStatus('agent', status.agentcurrent, status.agentversions)
applyBinaryStatus('tray', status.traycurrent, status.trayversions)
} catch (statusError) {
console.error(statusError)
}
})
function applyBinaryStatus(binkey, current, versions) {
const binary = binaryFor(binkey)
if (!binary) return
binary.current = current || null
binary.versions = Array.isArray(versions) ? versions : []
}
async function regenerate() {
if (key.value &&
!window.confirm('Regenerate the key? The current key stops working '
@@ -528,7 +729,10 @@ async function buildSiteAgent() {
const url = window.URL.createObjectURL(response.data)
const link = document.createElement('a')
link.href = url
link.download = 'shopdb-endpoint-agent.zip'
// Honor the server's versioned download name (shopdb-endpoint-agent-<ver>.zip)
// from Content-Disposition so operators see the agent version in the file
// name; fall back to the fixed name when the header is absent.
link.download = filenameFromResponse(response, 'shopdb-endpoint-agent.zip')
document.body.appendChild(link)
link.click()
link.remove()
@@ -548,6 +752,123 @@ async function buildSiteAgent() {
}
}
async function detectVersion(binkey, file) {
const form = new FormData()
form.append('file', file)
const response = await api.post(
`/endpoints/siteagent/binaries/${binkey}/detect`, form)
return response.data.data?.version || ''
}
function onBinaryFileChange(binkey, event) {
const binary = binaryFor(binkey)
if (!binary) return
const files = event.target.files
binary.file = files && files.length ? files[0] : ''
binary.error = ''
if (!binary.file) return
binary.detecting = true
detectVersion(binkey, binary.file)
.then((version) => {
if (version) binary.version = version
})
.catch((detectError) => {
console.error(detectError)
})
.finally(() => {
binary.detecting = false
})
}
function onLegacyFileChange(binkey, event) {
const binary = binaryFor(binkey)
if (!binary) return
const files = event.target.files
binary.legacyfile = files && files.length ? files[0] : ''
}
async function uploadBinary(binkey) {
const binary = binaryFor(binkey)
if (!binary || !binary.file || !binary.version) return
binary.busy = true
binary.message = ''
binary.error = ''
try {
const form = new FormData()
form.append('version', binary.version)
form.append('file', binary.file)
const response = await api.post(
`/endpoints/siteagent/binaries/${binkey}`, form)
const data = response.data.data || {}
applyBinaryStatus(binkey, data.current, data.versions)
binary.version = ''
binary.file = ''
binary.message = 'Uploaded and set as current.'
} catch (uploadError) {
binary.error = uploadError.response?.data?.data?.error?.message
|| (uploadError.response?.status === 403
? 'You do not have permission to upload agent binaries'
: 'Could not upload the binary')
console.error(uploadError)
} finally {
binary.busy = false
}
}
async function uploadLegacy(binkey) {
const binary = binaryFor(binkey)
if (!binary || !binary.legacyfile || !binary.legacyversion) return
binary.busy = true
binary.message = ''
binary.error = ''
try {
const form = new FormData()
form.append('version', binary.legacyversion)
form.append('file', binary.legacyfile)
form.append('makecurrent', 'false')
const response = await api.post(
`/endpoints/siteagent/binaries/${binkey}`, form)
const data = response.data.data || {}
applyBinaryStatus(binkey, data.current, data.versions)
binary.legacyversion = ''
binary.legacyfile = ''
binary.message = 'Legacy build staged for rollback.'
} catch (uploadError) {
binary.error = uploadError.response?.data?.data?.error?.message
|| (uploadError.response?.status === 403
? 'You do not have permission to upload agent binaries'
: 'Could not upload the legacy build')
console.error(uploadError)
} finally {
binary.busy = false
}
}
async function setCurrentBinary(binkey, version) {
const binary = binaryFor(binkey)
if (!binary) return
if (!window.confirm(`Make version ${version} the current staged build? `
+ 'New downloads will ship this version.')) {
return
}
binary.busy = true
binary.message = ''
binary.error = ''
try {
const response = await api.put(
`/endpoints/siteagent/binaries/${binkey}/current`, { version })
const data = response.data.data || {}
applyBinaryStatus(binkey, data.current, data.versions)
binary.message = `Version ${version} is now current.`
} catch (setError) {
binary.error = setError.response?.data?.data?.error?.message
|| 'Could not change the current version'
console.error(setError)
} finally {
binary.busy = false
}
}
function editConfiguration() {
router.push({ name: 'endpoints-monitors-settings' })
}
@@ -585,4 +906,72 @@ function editConfiguration() {
.settings-section > h3 { margin-bottom: 0.25rem; }
.info-list { margin: 0.25rem 0 0; padding-left: 1.25rem; }
.info-list li { margin-bottom: 0.5rem; }
.binary-block {
margin-top: 1.25rem;
padding-top: 1rem;
border-top: 1px solid var(--color-border, #333);
}
.binary-upload {
display: flex;
gap: 0.5rem;
align-items: center;
flex-wrap: wrap;
margin: 0.5rem 0;
}
.binary-upload .form-control[type="text"] { max-width: 12rem; }
.previous-versions { margin-top: 0.5rem; }
.collapsible { margin: 0.5rem 0; }
.collapsible > summary {
cursor: pointer;
list-style: none;
display: flex;
align-items: center;
gap: 0.4rem;
user-select: none;
}
.collapsible > summary::-webkit-details-marker { display: none; }
.collapsible > summary::before {
content: "\25be";
display: inline-block;
transform: rotate(-90deg);
transition: transform 0.15s ease;
color: var(--color-text-muted, #888);
}
.collapsible[open] > summary::before { transform: rotate(0deg); }
.collapsible > summary h4 { margin: 0; }
.field-warning {
margin: 0.25rem 0 0.5rem;
padding: 0.5rem 0.75rem;
border-left: 3px solid var(--color-warning, #b8860b);
background: rgba(184, 134, 11, 0.08);
font-size: 0.85rem;
font-weight: 600;
}
.btn-link {
background: none;
border: none;
color: var(--color-link, #4ea1ff);
cursor: pointer;
padding: 0;
font-size: 0.9rem;
text-decoration: underline;
}
.btn-link:disabled { color: var(--color-text-muted, #888); cursor: default; }
.versions-table {
width: 100%;
margin-top: 0.5rem;
border-collapse: collapse;
font-size: 0.85rem;
}
.versions-table th,
.versions-table td {
text-align: left;
padding: 0.3rem 0.5rem;
border-bottom: 1px solid var(--color-border, #333);
}
.current-tag {
font-size: 0.8rem;
color: var(--color-success, #4caf50);
font-weight: 600;
}
</style>

4
plugins/endpoints/frontend/views/EndpointsList.vue
View File

@@ -342,7 +342,9 @@ async function loadEndpoints() {
// navigates when it resolved to a computerid.
function openPc(item) {
if (item.computerid) {
router.push(`/pcs/${item.computerid}`)
// Tag the origin so PC Details and the edit form return here (Fleet)
// instead of the PCs list. Absent for the normal PCs-list entry path.
router.push(`/pcs/${item.computerid}?from=fleet`)
}
}

49
plugins/endpoints/migrations/versions/0011_endpoint_trayversion.py Normal file
View File

@@ -0,0 +1,49 @@
"""Cache the interactive tray app version on the endpoint state.
Adds endpointstates.trayversion: the file version of the ShopDB endpoint tray
app, reported by the agent alongside its own agentversion. The tray and the
agent service ship and update independently (they are separate exes and can be
pushed apart), so the tray version is stored as its own field rather than
assumed equal to the agent (AGENT-CONTRACT.md 2).
Nullable so an endpoint with no tray installed, or an older agent that predates
the field, reads as "unknown" rather than an empty string.
Idempotent: the column-add checks the live schema first, so a db.create_all()
site (which built the column from the model already) and a re-run both stay
clean on strict MySQL 8.
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'endpoints0011trayversion'
down_revision = 'endpoints0010renamepolicy'
branch_labels = None
depends_on = None
def _has_column(table_name, column_name):
from sqlalchemy import inspect
inspector = inspect(op.get_bind())
if not inspector.has_table(table_name):
return False
return any(col['name'] == column_name
for col in inspector.get_columns(table_name))
def upgrade():
if _has_column('endpointstates', 'trayversion'):
return
op.add_column(
'endpointstates',
sa.Column('trayversion', sa.String(length=40), nullable=True))
def downgrade():
if not _has_column('endpointstates', 'trayversion'):
return
op.drop_column('endpointstates', 'trayversion')

6
plugins/endpoints/models/endpoint.py
View File

@@ -59,6 +59,11 @@ class EndpointState(db.Model):
hostname = db.Column(db.String(100), nullable=True)
loggedinuser = db.Column(db.String(100), nullable=True)
agentversion = db.Column(db.String(40), nullable=True)
# The interactive tray app versions independently of the agent service, so
# it is reported and cached as its own field (AGENT-CONTRACT.md 2). Null
# when the endpoint has no tray installed or an older agent that predates
# the field.
trayversion = db.Column(db.String(40), nullable=True)
# Fast-moving metrics
lastboottime = db.Column(db.DateTime, nullable=True)
@@ -105,6 +110,7 @@ class EndpointState(db.Model):
'hostname': self.hostname,
'loggedinuser': self.loggedinuser,
'agentversion': self.agentversion,
'trayversion': self.trayversion,
'lastboottime': _utciso(self.lastboottime),
'uptimeseconds': self.uptimeseconds,
'idleseconds': self.idleseconds,

52
plugins/endpoints/plugin.py
View File

@@ -49,7 +49,7 @@ CURATED_EVENT_IDS = frozenset(EVENT_CATALOG)
KNOWN_PAYLOAD_KEYS = frozenset({
'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype',
'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress',
'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
'agentversion', 'trayversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces',
'scheduledtasks', 'applications', 'events',
})
@@ -167,19 +167,40 @@ class EndpointsPlugin(BasePlugin):
# running a staged script or pushing an installer is a strictly
# heavier grant than a routine service/process toggle.
('endpoints.deploy',
'Run staged scripts and push application installers to endpoints', 'endpoints'),
# Packaging the agent is a heavier grant than deploying to one, and
# it is not the same job either: the generated ZIP carries a live
# collector key stamped into its install script, so whoever holds
# this can mint a distributable artefact containing the fleet-wide
# command-channel credential. It was require_role('admin'), which
# this codebase otherwise reserves for the few genuinely
# role-shaped checks - and a role cannot be granted to one person
# for one job the way a permission can.
('endpoints.agentpackage',
'Generate and download an agent or adoption-tool package '
'(contains a live collector key)', 'endpoints'),
# Collector service scope for the endpoint agent. A managed token
'Run staged scripts and push application installers to endpoints', 'endpoints'),
# Packaging the agent is a heavier grant than deploying to one, and
# it is not the same job either: the generated ZIP carries a live
# collector key stamped into its install script, so whoever holds
# this can mint a distributable artefact containing the fleet-wide
# command-channel credential. It was require_role('admin'), which
# this codebase otherwise reserves for the few genuinely
# role-shaped checks - and a role cannot be granted to one person
# for one job the way a permission can.
# It now also governs SUPPLYING the binary, not just wrapping it.
# Before the agent store, the exe could only be staged by someone
# with filesystem access to instance/siteagent/; the upload routes
# put that behind this same grant, so a holder chooses the file
# every future package installs and runs as SYSTEM. Whoever grants
# this has to be able to see that from the description.
('endpoints.agentpackage',
'Upload the agent binary the fleet installs, and generate or '
'download an agent or adoption-tool package (supplies an '
'executable that runs as SYSTEM; the package contains a live '
'collector key)', 'endpoints'),
# Pushing an agent REPLACEMENT to running endpoints is a distinct,
# heavier job than either packaging (endpoints.agentpackage) or
# pushing an ordinary application installer (endpoints.deploy).
# Kept separate on purpose: the agent runs as SYSTEM and is the
# command channel itself, so replacing it is self-modifying the
# control plane. Folding it into endpoints.deploy would let anyone
# who can push an app silently swap the agent; folding it into
# endpoints.agentpackage would let a packager who only mints ZIPs
# also force a live fleet-wide swap. A future Operator role can
# hold upload without this, or this without upload.
('endpoints.agentupdate',
'Push agent/tray binary updates to running endpoints '
'(replaces the SYSTEM agent)', 'endpoints'),
# Collector service scope for the endpoint agent. A managed token
# scoped to ONLY this authorizes the endpoints collector and nothing
# else, so rotating/revoking it stops endpoint-agent traffic without
# touching computers/geenforce collectors. Minted from the settings
@@ -575,6 +596,7 @@ class EndpointsPlugin(BasePlugin):
},
'loggedinuser': {'type': 'string'},
'agentversion': {'type': 'string'},
'trayversion': {'type': 'string'},
'lastboottime': {'type': 'string', 'format': 'date-time'},
'uptimeseconds': {'type': 'integer'},
'idleseconds': {'type': 'integer'},
@@ -786,6 +808,8 @@ class EndpointsPlugin(BasePlugin):
state.loggedinuser = payload.get('loggedinuser')
if 'agentversion' in payload:
state.agentversion = payload.get('agentversion')
if 'trayversion' in payload:
state.trayversion = payload.get('trayversion')
if 'lastboottime' in payload:
state.lastboottime = _parse_dt(payload.get('lastboottime'))
if 'uptimeseconds' in payload:

42
scripts/Start-ShopDBDev.ps1
View File

@@ -85,27 +85,27 @@ function Clear-InterruptedRun {
}
}
# A directory that exists but has no valid .git is a broken/partial clone -
# USUALLY. It is also what -TargetDir pointed at somebody's Documents folder
# looks like, and this ran Remove-Item -Recurse -Force on it without asking.
# For a script distributed as "save it anywhere and run it", a mistyped
# -TargetDir must not be able to delete a user's files.
#
# Only reclaim a directory this script can show it owns: empty, or carrying
# the marker written after a successful clone. Anything else stops.
if ((Test-Path $Target) -and -not (Test-Path (Join-Path $Target '.git'))) {
$ours = Test-Path (Join-Path $Target $script:CloneMarker)
$empty = -not (Get-ChildItem -LiteralPath $Target -Force |
Select-Object -First 1)
if ($ours -or $empty) {
Write-Host "==> Removing incomplete clone at $Target" -ForegroundColor Cyan
Remove-Item -Path $Target -Recurse -Force
}
else {
throw ("$Target already exists, is not empty, and was not created " +
"by this launcher. Refusing to delete it. Pass -TargetDir " +
'pointing somewhere else, or remove it yourself if you are sure.')
}
# A directory that exists but has no valid .git is a broken/partial clone -
# USUALLY. It is also what -TargetDir pointed at somebody's Documents folder
# looks like, and this ran Remove-Item -Recurse -Force on it without asking.
# For a script distributed as "save it anywhere and run it", a mistyped
# -TargetDir must not be able to delete a user's files.
#
# Only reclaim a directory this script can show it owns: empty, or carrying
# the marker written after a successful clone. Anything else stops.
if ((Test-Path $Target) -and -not (Test-Path (Join-Path $Target '.git'))) {
$ours = Test-Path (Join-Path $Target $script:CloneMarker)
$empty = -not (Get-ChildItem -LiteralPath $Target -Force |
Select-Object -First 1)
if ($ours -or $empty) {
Write-Host "==> Removing incomplete clone at $Target" -ForegroundColor Cyan
Remove-Item -Path $Target -Recurse -Force
}
else {
throw ("$Target already exists, is not empty, and was not created " +
"by this launcher. Refusing to delete it. Pass -TargetDir " +
'pointing somewhere else, or remove it yourself if you are sure.')
}
}
# A clone created by an elevated (Administrator) run is owned by

3
scripts/build-channel-installer.ps1
View File

@@ -6,7 +6,7 @@
Produces ONE self-contained installer package for the requested channel:
stable -> built from 'main', no DEV badge (SHOPDB_ENV_LABEL empty).
edge -> built from 'main' (the trunk), red DEV badge (SHOPDB_ENV_LABEL=DEV).
edge -> built from 'main' (the trunk), red DEV badge (SHOPDB_ENV_LABEL=DEV).
This is the distributable counterpart to scripts/dev-bootstrap.ps1 (which
stands a dev box up FROM the repo). Here the operator has no repo: the script
@@ -102,6 +102,7 @@ function Resolve-StableRef {
Write-Warning 'No v* tag in this repo; building stable from main.'
return 'main'
}
}
function Assert-LastExit([string]$What) {
if ($LASTEXITCODE -ne 0) { throw "$What failed (exit $LASTEXITCODE)" }

4
scripts/check-naming-and-style.sh
View File

@@ -35,6 +35,10 @@ EXCLUDES=(
--exclude-dir=.git
--exclude-dir=versions
--exclude-dir=staticdocs
# Local, gitignored scratch notes (see .gitignore devnotes/). Not product
# source, never committed, and free to hold pasted logs full of em-dashes
# and arrows - grep does not consult .gitignore, so exclude it explicitly.
--exclude-dir=devnotes
)
INCLUDES_CODE=(

30
scripts/dev-bootstrap.ps1
View File

@@ -12,7 +12,7 @@
passed, because the app itself runs entirely in the container.
2. Picks a channel:
stable -> pulls the 'stable' image (built from main), no DEV badge.
edge -> pulls the 'edge' image (built from main), red DEV badge.
edge -> pulls the 'edge' image (built from main), red DEV badge.
The two channels use different compose project names, host ports, and
named volumes, so they run side by side without colliding.
3. Generates a .env for the channel (dev-grade secrets, NOT for production)
@@ -282,20 +282,20 @@ try {
} else {
# Dev-grade secrets: generated locally, never for production. A real
# deploy fills .env from the wiki runbook, not from this script.
#
# Get-Random is System.Random: a 32-bit seed, and all four values came
# off one stream, so recovering the seed from any one of them recovers
# the other three. These sign JWTs for a stack a pilot may point at a
# real imported dataset, so they come from the CSPRNG instead. "Dev
# grade" is about rotation and provenance, not about being guessable.
$secret = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$jwt = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$dbPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
$rootPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
#
# Get-Random is System.Random: a 32-bit seed, and all four values came
# off one stream, so recovering the seed from any one of them recovers
# the other three. These sign JWTs for a stack a pilot may point at a
# real imported dataset, so they come from the CSPRNG instead. "Dev
# grade" is about rotation and provenance, not about being guessable.
$secret = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$jwt = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$dbPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
$rootPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
$apiPort = $config.ApiPort
$mysqlPort = $config.MysqlPort
$image = "$imageRepo`:$($config.ImageTag)"

149
scripts/site_imports/approval.py Normal file
View File

@@ -0,0 +1,149 @@
"""Reusable approval / promotion gate for site-import writers.
IMPORT-AGNOSTIC. This turns a dry-run plan into an "approval packet": a frozen
summary of what an import WOULD do, plus a content hash of the exact plan it
describes. A writer refuses to run unless it is handed a packet whose hash still
matches the plan in front of it. That is the whole mechanism: a human approves a
specific, frozen plan, and the writer can prove nothing drifted since.
The same packet shape serves every importer, and it is deliberately the seam the
undo work will reuse: a writer records the packet's batch id alongside the
journal it writes, so approval, write, and undo all reference one identity.
A packet is a plain dict (JSON-serialisable) so it can be printed, saved next to
an export, diffed between runs, and shown in a UI without any of this code:
{
'kind': caller-supplied label, e.g. 'winc-machine-adoption'
'planhash': sha256 of the canonicalised plan rows
'rowcount': number of rows in the plan
'counts': caller-supplied summary dict (what the report shows)
'needscreate': { group -> { name -> count } } # requires acknowledgment
'unresolved': { group -> { value -> count } } # blocks approval
'blocking': True if any unresolved entries exist
}
The writer's contract, in order:
1. rebuild the plan from the same inputs,
2. build a packet from it,
3. compare planhash to the approved packet - abort on mismatch,
4. refuse if packet['blocking'],
5. require explicit acknowledgment for every needscreate group,
6. only then write, stamping packet['batchid'] onto the journal.
"""
import hashlib
import json
import uuid
from collections import OrderedDict
from scripts.site_imports.resolver import tally, NEEDS_CREATE, UNRESOLVED
def plan_hash(plan):
"""Stable sha256 over the plan's rows.
Canonicalised by serialising each row as sorted-key JSON so the hash depends
only on content, not on dict ordering or whitespace. Two runs that produce
the same decisions produce the same hash; any changed decision changes it,
which is exactly what the writer checks before it touches the database.
"""
hasher = hashlib.sha256()
for row in plan:
canonical = json.dumps(row, sort_keys=True, default=str,
separators=(',', ':'))
hasher.update(canonical.encode('utf-8'))
hasher.update(b'\n')
return hasher.hexdigest()
def build_packet(kind, plan, counts, groups):
"""Build an approval packet from a plan.
kind a short label identifying the importer, stored on the packet.
plan the list of per-row dicts the writer will act on.
counts the caller's summary dict (shown to the human as-is).
groups how to read each field's status/value columns out of a row:
[ (group_name, status_column, value_column), ... ]
e.g. ('businessunit', 'businessunit_status', 'raw_eq1'). The value
column is what a human would act on - the raw export value for
UNRESOLVED (so they know what to alias) and the canonical name for
NEEDS_CREATE (so they know what would be created).
needscreate is keyed on the canonical name column when present; callers pass
the *_name column for that via the fourth tuple element when it differs from
the value column.
"""
needscreate = OrderedDict()
unresolved = OrderedDict()
for group in groups:
group_name, status_column, value_column = group[0], group[1], group[2]
name_column = group[3] if len(group) > 3 else value_column
unresolved_hits = tally(plan, status_column, value_column, UNRESOLVED)
needs_hits = tally(plan, status_column, name_column, NEEDS_CREATE)
if unresolved_hits:
unresolved[group_name] = unresolved_hits
if needs_hits:
needscreate[group_name] = needs_hits
return OrderedDict([
('kind', kind),
('batchid', str(uuid.uuid4())),
('planhash', plan_hash(plan)),
('rowcount', len(plan)),
('counts', OrderedDict(counts)),
('needscreate', needscreate),
('unresolved', unresolved),
('blocking', bool(unresolved)),
])
def packet_matches_plan(packet, plan):
"""True if the approved packet still describes this exact plan."""
return packet.get('planhash') == plan_hash(plan)
def format_packet(packet):
"""Human-readable rendering of a packet for the console or a UI panel."""
lines = []
lines.append('APPROVAL PACKET: %s' % packet.get('kind', '?'))
lines.append(' batch id : %s' % packet.get('batchid', '?'))
lines.append(' plan hash: %s' % packet.get('planhash', '?'))
lines.append(' rows : %d' % packet.get('rowcount', 0))
needscreate = packet.get('needscreate') or {}
if needscreate:
for group, items in needscreate.items():
total = sum(items.values())
lines.append('')
lines.append(' WARNING: this import would CREATE %d %s(s). '
'Confirm these are correct; you can edit them later '
'in Settings:' % (len(items), group))
for name, count in items.items():
lines.append(' + %-28s (%d rows)' % (name, count))
lines.append(' total new %s rows: %d' % (group, total))
unresolved = packet.get('unresolved') or {}
if unresolved:
for group, items in unresolved.items():
lines.append('')
lines.append(' BLOCKING: %d unresolved %s value(s) - add an alias '
'for each before importing:' % (len(items), group))
for value, count in items.items():
lines.append(' ? %-28s (%d rows)' % (value, count))
lines.append('')
lines.append(' status : %s' % (
'BLOCKED (resolve the values above)' if packet.get('blocking')
else 'READY for acknowledgment'))
return '\n'.join(lines)
def save_packet(packet, path):
with open(path, 'w', encoding='utf-8') as handle:
json.dump(packet, handle, indent=2)
def load_packet(path):
with open(path, encoding='utf-8') as handle:
return json.load(handle)

51
scripts/site_imports/clone_import_db.ps1 Normal file
View File

@@ -0,0 +1,51 @@
# Clone the live shopdb_flask database into a throwaway copy for import testing.
#
# Runs entirely inside the db container so the root password is read from the
# container's own environment and never typed on a command line or logged. The
# clone is a full structural + data copy, so a dry run or a writer pointed at it
# behaves exactly as it would against production - but nothing it does can touch
# the real data.
#
# pwsh scripts/site_imports/clone_import_db.ps1
# pwsh scripts/site_imports/clone_import_db.ps1 -Target shopdb_flask_import
#
# Point a tool at the clone by overriding DATABASE_URL for that one run, e.g.
# docker compose exec -e DATABASE_URL="mysql+pymysql://root:PW@db:3306/shopdb_flask_import?charset=utf8mb4" api ...
# (the winc dry run also honours DATABASE_URL, so --export works unchanged).
param(
[string]$Source = 'shopdb_flask',
[string]$Target = 'shopdb_flask_import'
)
$ErrorActionPreference = 'Stop'
Write-Host "Cloning '$Source' -> '$Target' inside the db container..."
# All work happens in the container. $MYSQL_ROOT_PASSWORD is already set there.
# MYSQL_PWD is honoured by both mysql and mysqldump, which keeps the password
# off the argument list (no -p on the command line to leak into ps or logs).
# Backticks around identifiers are MySQL's; placeholders are filled in by
# PowerShell below so the shell here sees plain literal database names.
$remote = @'
set -e
export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"
mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS \`__TARGET__\`; CREATE DATABASE \`__TARGET__\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
mysqldump -h 127.0.0.1 -u root --single-transaction --routines --triggers __SOURCE__ | mysql -h 127.0.0.1 -u root __TARGET__
mysql -h 127.0.0.1 -u root -N -e "SELECT CONCAT('assets in clone: ', COUNT(*)) FROM __TARGET__.assets;"
'@
$remote = $remote.Replace('__TARGET__', $Target).Replace('__SOURCE__', $Source)
# Strip CR so the container's sh sees clean LF-only lines (this file is CRLF on
# a Windows checkout, and sh treats a trailing \r as part of the argument).
$remote = $remote.Replace("`r", '')
# The db service name in docker-compose.yml is 'db'.
docker compose exec -T db sh -c $remote
if ($LASTEXITCODE -ne 0) {
throw "Clone failed (exit $LASTEXITCODE)."
}
Write-Host "Clone complete: $Target"

267
scripts/site_imports/journal.py Normal file
View File

@@ -0,0 +1,267 @@
"""Shared, import-agnostic journal + undo for reversible bulk imports.
IMPORT-AGNOSTIC. Nothing here knows about WINC, Maximo, machines, or any one
source. An import writer records the primitive mutations it makes - "set column
X on row Y of table T from old to new" (update) or "insert row Y into table T"
(insert) - each tied to a batch id, and a generic undo replays them in reverse.
That is the whole mechanism: importers emit primitives; undo reverses primitives.
Identity: one value threads approval, write, and undo. The approval packet's
`batchid` becomes the importbatches primary key, becomes the journal foreign key,
becomes the argument the "Undo import" button passes. Nothing to reconcile.
Contract (extends the approval packet's 6-step writer contract):
open_batch() -> after the packet passes, write the importbatches row.
record_update()/record_insert() -> append a journal entry in the SAME
transaction as the mutation it describes (ledger rule: the
entry and the change it records commit together).
undo_batch() -> reverse-replay this batch's entries, conservatively.
Undo is deliberately conservative and never destructive beyond what the import
owns:
- update reverts a cell ONLY if it still holds the value we wrote; a value a
human changed since is left as-is and reported as a conflict (never clobbered).
- insert hard-deletes ONLY the row this batch created, and only when nothing
outside the batch still references it; otherwise it is left and reported.
All access goes through a SQLAlchemy connection executing parameterised text, so
this reuses the same connection an importer already opened and pulls in no ORM
models - keeping it a true platform primitive any importer can share.
"""
from collections import namedtuple
from datetime import datetime, timezone
BATCH_APPLIED = 'applied'
BATCH_UNDONE = 'undone'
OP_UPDATE = 'update'
OP_INSERT = 'insert'
# Result of an undo pass, for a report and a UI.
UndoResult = namedtuple('UndoResult', ('batchid', 'reverted', 'conflicts'))
# A single thing undo could not safely reverse.
Conflict = namedtuple('Conflict', ('operation', 'table', 'targetid',
'column', 'reason'))
def _utcnow():
return datetime.now(timezone.utc).replace(tzinfo=None)
def _as_text(value):
"""Store every journalled value as text so a pre/post image compares the
same way it will read back, regardless of the column's declared type."""
return None if value is None else str(value)
def open_batch(connection, packet, createdby=None):
"""Write the importbatches row for an approved packet.
Call once, after the packet has been verified against the current plan and
acknowledged. Returns the batchid so the caller stamps it on every journal
entry it then records. Import-agnostic: the packet's own fields carry all
the identity this needs.
"""
from sqlalchemy import text
connection.execute(text(
'INSERT INTO importbatches '
'(batchid, kind, planhash, rowcount, createdat, createdby, status) '
'VALUES (:batchid, :kind, :planhash, :rowcount, :createdat, '
':createdby, :status)'),
{
'batchid': packet['batchid'],
'kind': packet.get('kind'),
'planhash': packet.get('planhash'),
'rowcount': packet.get('rowcount', 0),
'createdat': _utcnow(),
'createdby': createdby,
'status': BATCH_APPLIED,
})
return packet['batchid']
def record_update(connection, batchid, sequence, table, targetid, column,
oldvalue, newvalue):
"""Append an 'update' journal entry describing one cell change.
Record the pre-image (oldvalue) you read BEFORE applying the change and the
post-image (newvalue) you are about to write, in the same transaction as the
UPDATE itself.
"""
_insert_entry(connection, batchid, sequence, OP_UPDATE, table, targetid,
column, oldvalue, newvalue)
def record_insert(connection, batchid, sequence, table, targetid):
"""Append an 'insert' journal entry for a row this batch created.
Only ever emitted for a needs-create row the writer itself inserted, so undo
can hard-delete strictly what the import created.
"""
_insert_entry(connection, batchid, sequence, OP_INSERT, table, targetid,
None, None, None)
def _insert_entry(connection, batchid, sequence, operation, table, targetid,
column, oldvalue, newvalue):
from sqlalchemy import text
connection.execute(text(
'INSERT INTO importjournalentries '
'(batchid, sequence, operation, targettable, targetid, columnname, '
'oldvalue, newvalue, isreverted) '
'VALUES (:batchid, :sequence, :operation, :targettable, :targetid, '
':columnname, :oldvalue, :newvalue, 0)'),
{
'batchid': batchid,
'sequence': sequence,
'operation': operation,
'targettable': table,
'targetid': targetid,
'columnname': column,
'oldvalue': _as_text(oldvalue),
'newvalue': _as_text(newvalue),
})
def undo_batch(connection, batchid, references=None):
"""Reverse-replay an applied batch's journal. Returns an UndoResult.
references (optional): { table -> [(reftable, refcolumn), ...] } describing
which columns in other tables point at a created row's id. Used to decide
whether an inserted row is safe to hard-delete: if any reference OUTSIDE
this batch's own updates still points at it, the row is left and reported as
a conflict rather than deleted. Omit it to require that the batch's own
update-reverts have already cleared every reference (the common case, since a
created businessunit is only ever pointed at by the same batch's asset
updates).
Idempotent: a batch already 'undone' is a no-op. Entries are walked in
reverse sequence so inserts are removed only after the updates that pointed
at them are reverted.
"""
from sqlalchemy import text
batch = connection.execute(text(
'SELECT status FROM importbatches WHERE batchid = :batchid'),
{'batchid': batchid}).fetchone()
if batch is None:
raise ValueError('unknown batchid: %s' % batchid)
if batch[0] == BATCH_UNDONE:
return UndoResult(batchid, 0, [])
entries = connection.execute(text(
'SELECT entryid, sequence, operation, targettable, targetid, '
'columnname, oldvalue, newvalue FROM importjournalentries '
'WHERE batchid = :batchid AND isreverted = 0 '
'ORDER BY sequence DESC, entryid DESC'),
{'batchid': batchid}).fetchall()
reverted = 0
conflicts = []
for row in entries:
(entryid, _sequence, operation, table, targetid, column,
oldvalue, newvalue) = row
if operation == OP_UPDATE:
conflict = _revert_update(connection, table, targetid, column,
oldvalue, newvalue)
elif operation == OP_INSERT:
conflict = _revert_insert(connection, batchid, table, targetid,
references)
else:
conflict = Conflict(operation, table, targetid, column,
'unknown operation')
if conflict is None:
connection.execute(text(
'UPDATE importjournalentries SET isreverted = 1 '
'WHERE entryid = :entryid'), {'entryid': entryid})
reverted += 1
else:
conflicts.append(conflict)
connection.execute(text(
'UPDATE importbatches SET status = :status, undoneat = :undoneat '
'WHERE batchid = :batchid'),
{'status': BATCH_UNDONE, 'undoneat': _utcnow(), 'batchid': batchid})
return UndoResult(batchid, reverted, conflicts)
def _revert_update(connection, table, targetid, column, oldvalue, newvalue):
"""Revert one cell to its pre-image, but only if it still holds what we
wrote. Returns a Conflict if a human changed it since, else None."""
from sqlalchemy import text
pk = _primary_key(connection, table)
current = connection.execute(text(
'SELECT %s FROM %s WHERE %s = :targetid' % (column, table, pk)),
{'targetid': targetid}).fetchone()
if current is None:
return Conflict(OP_UPDATE, table, targetid, column, 'row missing')
if _as_text(current[0]) != _as_text(newvalue):
return Conflict(OP_UPDATE, table, targetid, column,
'value changed since import')
connection.execute(text(
'UPDATE %s SET %s = :oldvalue WHERE %s = :targetid'
% (table, column, pk)),
{'oldvalue': oldvalue, 'targetid': targetid})
return None
def _revert_insert(connection, batchid, table, targetid, references):
"""Hard-delete a row this batch created, only if nothing outside the batch
still references it. Returns a Conflict when it is not safe, else None."""
from sqlalchemy import text
pk = _primary_key(connection, table)
for reftable, refcolumn in (references or {}).get(table, []):
referencing = connection.execute(text(
'SELECT COUNT(*) FROM %s WHERE %s = :targetid'
% (reftable, refcolumn)), {'targetid': targetid}).fetchone()[0]
if referencing:
return Conflict(OP_INSERT, table, targetid, None,
'still referenced by %s.%s' % (reftable, refcolumn))
connection.execute(text(
'DELETE FROM %s WHERE %s = :targetid' % (table, pk)),
{'targetid': targetid})
return None
def _primary_key(connection, table):
"""Single-column primary key name for a table, via the live schema."""
from sqlalchemy import inspect
pk = inspect(connection).get_pk_constraint(table)
columns = pk.get('constrained_columns') or []
if len(columns) != 1:
raise ValueError(
'journal undo needs a single-column primary key on %s' % table)
return columns[0]
def format_undo_result(result):
"""Human-readable rendering of an UndoResult for the console or a UI panel."""
lines = []
lines.append('UNDO batch %s' % result.batchid)
lines.append(' reverted : %d entries' % result.reverted)
if result.conflicts:
lines.append(' conflicts: %d (left as-is, not reverted)'
% len(result.conflicts))
for conflict in result.conflicts:
target = conflict.table
if conflict.column:
target = '%s.%s' % (conflict.table, conflict.column)
lines.append(' ! %-8s %s #%s - %s' % (
conflict.operation, target, conflict.targetid,
conflict.reason))
else:
lines.append(' conflicts: none')
return '\n'.join(lines)

134
scripts/site_imports/resolver.py Normal file
View File

@@ -0,0 +1,134 @@
"""Shared four-state resolver vocabulary for site-import dry runs.
IMPORT-AGNOSTIC. Nothing here knows about machines, eq1, locations, or any one
site. Every site importer maps a raw exported value onto a database row, and
every one of those mappings lands in exactly one of four states:
RESOLVED the value maps to a row that already exists; safe to set.
BLANK there is nothing to set (empty cell, or a deliberately
unmappable value); leave the target field as-is.
NEEDS_CREATE the value maps to a valid canonical name, but that row does
not exist yet; a writer may create it, WITH acknowledgment.
UNRESOLVED the value has no mapping at all; a human must add one before
any writer runs. This is the state that blocks approval.
Keeping this vocabulary in one place is what lets a promotion/approval helper,
an undo journal, and any number of importers all speak the same language. Add
new importers by writing their lookups and alias maps; reuse these primitives to
turn a raw value into a Resolution.
"""
from collections import OrderedDict, namedtuple
# The four terminal states a single field resolution can end in.
RESOLVED = 'resolved'
BLANK = 'blank'
NEEDS_CREATE = 'needs-create'
UNRESOLVED = 'unresolved'
# status: one of the four above.
# id: the database id when RESOLVED, else None.
# name: the canonical target name when known (RESOLVED / NEEDS_CREATE), else
# None. Carried so callers can report and, later, create by name.
Resolution = namedtuple('Resolution', ('status', 'id', 'name'))
def norm(value):
"""Lowercase, stripped form used for every name comparison and alias key.
Centralised so a lookup table, an alias map, and the value being resolved
are always normalised the same way. If this ever changes, it changes for
everyone at once, which is the point.
"""
return (value or '').strip().lower()
def load_lookup(connection, table, id_column, name_column, display=None):
"""norm(name) -> id for every row of any table. One loader, every importer.
This is the "match against what is actually in the database" primitive. It
reads the live table at call time, so a value a human adds in Settings is
visible on the next run with no code change: a name that exists resolves,
one that does not is NEEDS_CREATE or UNRESOLVED. Every site importer maps its
exported values onto some reference table (locations, businessunits, vendors,
...) and they should all load that table the same way rather than each
hand-rolling a SELECT.
Pass a dict as `display` to also collect norm(name) -> original-cased name,
so a report can echo the name the way the database spells it rather than the
normalised key.
First writer wins on a duplicate normalised name, which is deterministic and
matches how the importers resolve: the ambiguity is a data problem to report,
not something the loader should silently reorder.
"""
from sqlalchemy import text
out = {}
rows = connection.execute(text(
'SELECT %s, %s FROM %s' % (id_column, name_column, table)))
for ident, name in rows:
key = norm(name)
if key:
out.setdefault(key, ident)
if display is not None:
display.setdefault(key, (name or '').strip())
return out
def resolve_name(name, lookup_by_name, missing=UNRESOLVED):
"""Resolve a value by direct name match against a lookup table.
lookup_by_name maps norm(name) -> database id. A blank name is BLANK. A hit
is RESOLVED with the id. A miss is `missing` - callers pass UNRESOLVED when a
human must supply a mapping, or NEEDS_CREATE when the name is trusted and a
writer may create it.
"""
if not (name or '').strip():
return Resolution(BLANK, None, None)
ident = lookup_by_name.get(norm(name))
if ident is not None:
return Resolution(RESOLVED, ident, name)
return Resolution(missing, None, name)
def resolve_alias(raw, alias_map, lookup_by_name, missing=UNRESOLVED):
"""Resolve a dirty raw value through an alias map, then a lookup table.
The alias-first path: a raw export value (possibly truncated or oddly cased)
is normalised and looked up in alias_map to get a canonical name, which is
then resolved against lookup_by_name.
raw is blank -> BLANK (nothing to set)
raw not in alias_map -> missing (default UNRESOLVED: add an alias)
alias maps to None -> BLANK (deliberately unmappable value)
canonical exists in lookup -> RESOLVED
canonical not yet in lookup -> NEEDS_CREATE
"""
if not (raw or '').strip():
return Resolution(BLANK, None, None)
key = norm(raw)
if key not in alias_map:
return Resolution(missing, None, None)
canonical = alias_map[key]
if canonical is None:
return Resolution(BLANK, None, None)
return resolve_name(canonical, lookup_by_name, missing=NEEDS_CREATE)
def tally(plan, status_column, value_column, status):
"""Count rows in a plan whose status_column equals `status`, by value.
Returns an OrderedDict of value -> count, highest count first. Shared by the
dry-run report and the approval packet so the numbers a human reads and the
numbers a writer enforces come from one place.
"""
counts = {}
for row in plan:
if row.get(status_column) == status:
value = row.get(value_column) or ''
counts[value] = counts.get(value, 0) + 1
ordered = OrderedDict()
for value, count in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
ordered[value] = count
return ordered

159
scripts/site_imports/suggest.py Normal file
View File

@@ -0,0 +1,159 @@
"""Conservative, explainable match suggestions for UNRESOLVED values.
IMPORT-AGNOSTIC and REPORT-ONLY. This never resolves anything and never writes.
Given a raw value that the normal rules could not map, it proposes zero or more
candidate names that already exist in a lookup table, each with a reason and a
confidence, so a human can accept one into that importer's alias map. Nothing
here is ever applied automatically - a silent wrong match is the worst failure
mode in dirty asset data, so every suggestion has to be acknowledged.
Two tiers, tried in order, stopping at the first that yields anything:
1. Bounded deterministic transforms (confidence 'high'). These are rules that
fit the shapes actually seen in the data - a bay code with a slash is two
bays (T6/U6 -> T6, U6), a parenthetical is a note (LD9W(GE90) -> LD9W), a
decimal is a sub-bay (M6.3 -> M6), a trailing range is its first member
(L1-4 -> L1). Each transform is a named rule, not a fuzzer, so the reason
printed next to a suggestion tells a human exactly why it was proposed.
2. A single edit-distance-1 fallback (confidence 'low'). Only if no transform
matched, and only for a one-character difference against an existing name.
Bounded to distance 1 on purpose: it catches a typo or a dropped letter and
nothing looser, so it cannot quietly invent a plausible-looking wrong bay.
The caller supplies the same normalisation and the lookup it resolves against, so
suggestions are always checked against what is really in the database.
"""
import re
from collections import namedtuple
from scripts.site_imports.resolver import norm
# candidate: the existing name we think the raw value meant.
# reason: which rule proposed it (shown to the human).
# confidence: 'high' for a deterministic transform, 'low' for edit-distance.
Suggestion = namedtuple('Suggestion', ('candidate', 'reason', 'confidence'))
_PAREN = re.compile(r'\s*\([^)]*\)')
def _candidates_from_transforms(stripped):
"""Yield (transformed_name, reason) from the bounded deterministic rules.
Operates on the value AFTER the importer has removed its own prefix (e.g.
WM-). Each rule yields plain names; the caller checks each against the live
lookup, so a rule that produces a non-existent name simply contributes no
suggestion.
"""
# A parenthetical note: LD9W(GE90) -> LD9W
if '(' in stripped:
yield (_PAREN.sub('', stripped).strip(), 'dropped parenthetical note')
# A slash means two bays: T6/U6 -> T6 and U6. A bare second half (9/10)
# inherits the first half's letter prefix (T9/10 -> T9, T10).
if '/' in stripped:
left, right = (part.strip() for part in stripped.split('/', 1))
if left:
yield (left, 'first of a slash-joined pair')
if right:
yield (right, 'second of a slash-joined pair')
prefix = re.match(r'^([A-Za-z]+)', left)
if prefix and re.match(r'^\d', right):
yield (prefix.group(1) + right,
'second of a slash-joined pair, prefix carried')
# A decimal sub-bay: M6.3 -> M6
if '.' in stripped:
yield (stripped.split('.', 1)[0].strip(), 'dropped decimal sub-bay')
# A trailing numeric range: L1-4 -> L1
dash = re.match(r'^([A-Za-z]*\d+)-\d+$', stripped)
if dash:
yield (dash.group(1), 'first of a numeric range')
def _edit_distance_at_most_one(left, right):
"""True if `left` and `right` differ by at most one insert/delete/replace.
Bounded on purpose. A length gap over one is an immediate no; otherwise walk
both once and allow a single mismatch. Cheap and strict - it is a typo
catcher, not a similarity search.
"""
if left == right:
return True
length_left, length_right = len(left), len(right)
if abs(length_left - length_right) > 1:
return False
if length_left == length_right:
return sum(a != b for a, b in zip(left, right)) == 1
# Ensure left is the shorter one, then try to match with one insertion.
if length_left > length_right:
left, right = right, left
length_left, length_right = length_right, length_left
index_left = index_right = 0
skipped = False
while index_left < length_left and index_right < length_right:
if left[index_left] == right[index_right]:
index_left += 1
index_right += 1
elif skipped:
return False
else:
skipped = True
index_right += 1
return True
# A low-confidence edit-distance pass can match many short bay codes at once,
# which is noise, not help. Cap how many it emits so a human sees the few most
# plausible typo-fixes and nothing drowns the high-confidence transforms.
_MAX_LOW_CONFIDENCE = 3
def suggest(raw_value, stripped_value, lookup_by_name, display=None):
"""Suggestions for one unresolved value. Empty list means "no idea".
raw_value the original export string (for reporting only).
stripped_value the value after the importer removed its own prefix; this is
what the transforms and the edit-distance search operate on.
lookup_by_name norm(name) -> id, the live table to match against.
display optional norm(name) -> original-cased name, so a suggestion
echoes the database's spelling instead of the normalised key.
Deterministic transforms first; only if none of them hits an existing name
does the edit-distance-1 fallback run, and that fallback is capped. High-
confidence suggestions are never diluted by low-confidence ones.
"""
display = display or {}
def spell(key):
return display.get(key, key)
results = []
seen = set()
for candidate, reason in _candidates_from_transforms(stripped_value):
candidate = (candidate or '').strip()
if not candidate:
continue
key = norm(candidate)
if key in lookup_by_name and key not in seen:
seen.add(key)
results.append(Suggestion(spell(key), reason, 'high'))
if results:
return results
target = norm(stripped_value)
if not target:
return results
low = []
for existing_key in lookup_by_name:
if existing_key not in seen and _edit_distance_at_most_one(
target, existing_key):
seen.add(existing_key)
low.append(Suggestion(spell(existing_key),
'one-character difference', 'low'))
# Only surface the capped set; too many one-off matches is noise.
if len(low) <= _MAX_LOW_CONFIDENCE:
results.extend(sorted(low))
return results

67
scripts/site_imports/winc/README.md Normal file
View File

@@ -0,0 +1,67 @@
# WINC machine-adoption import (CLI)
Adopts machines from a Maximo asset export (validated against the WINC / West
Manufacturing site). It only updates machines that already exist; it never
creates canonical per-table rows. Every commit is journaled so it can be undone
as one batch.
Run all commands from the repo root. Each reads `.env` and connects to the
configured database.
## 1. Preview (read-only)
```
python -m scripts.site_imports.winc.dry_run --export EXPORT.csv
```
| switch | what it does |
| --- | --- |
| `--export PATH` | the Maximo ASSET export CSV to analyze (required) |
| `--rows` | print the per-row plan, not just the summary |
| `--csv PATH` | write the full plan to a CSV so runs can be diffed |
Nothing is written. Use this to see what would change and which values are
unresolved before committing.
## 2. Commit (writes, reversibly)
```
python -m scripts.site_imports.winc.writer --export EXPORT.csv --commit --acknowledge businessunit
```
| switch | what it does |
| --- | --- |
| `--export PATH` | the same export CSV (required) |
| `--commit` | actually write; without it this is a dry run |
| `--acknowledge GROUP` | approve a needs-create group (repeatable), e.g. `businessunit` |
| `--createdby NAME` | who approved the import; stamped on the batch |
On success it prints the new `batchid` and the exact undo command.
## 3. List batches
```
python -m scripts.site_imports.winc.batches
```
| switch | what it does |
| --- | --- |
| `--limit N` | show at most N batches, newest first (default 50) |
| `--json` | emit JSON instead of a table |
Read-only. Use it to find the `batchid` to undo.
## 4. Undo a batch
```
python -m scripts.site_imports.winc.undo --batch BATCHID --commit
```
| switch | what it does |
| --- | --- |
| `--batch BATCHID` | the batch to reverse (required) |
| `--commit` | actually reverse; without it this is a no-op |
Reversal is conservative: a cell a human changed since the import is left as-is
and reported, and a created business unit is deleted only when nothing outside
the batch still references it.

0
scripts/site_imports/winc/__init__.py Normal file
View File

67
scripts/site_imports/winc/aliases.py Normal file
View File

@@ -0,0 +1,67 @@
"""Alias and normalization tables for the WINC machine adoption dry-run.
This file is DATA, not logic. It is the single place a human edits when the
analyzer reports an unresolved value. Keep it boring: plain dicts, one entry per
line, lowercase keys. The analyzer lowercases and strips before it looks a value
up here, so keys here must be lowercase and already stripped.
Two independent problems get solved here:
1. eq1 -> business unit. The May export's eq1 column is a free-text value-stream
label (LG PARTS, SM SPOOLS, Large Spoo, HPTCR ...). It is dirty: truncated at
ten characters, inconsistently cased, and it does NOT match anything in the
live businessunits table (that table is a list of offices auto-created by an
earlier printer import). So every eq1 value must be mapped by hand to the
canonical business-unit name we want on the asset. A canonical name that does
not yet exist in the database is reported by the analyzer as "would need to
be created" - the analyzer never invents one and never writes one.
2. location code overrides. Most WM-<bay> codes resolve by simply dropping the
WM- prefix and matching a location by name (WM-J3 -> J3). A handful do not,
because the bay code carries a slash, a note in parentheses, or a room name
the location table spells differently. Those exceptions - and only those -
live here, mapping the RAW export location string to the exact locationname
already in the database.
"""
# eq1 (lowercased, stripped) -> canonical business unit name.
# The value on the right is what we intend to store; the analyzer checks whether
# it already exists in businessunits and flags the ones that do not.
EQ1_TO_BUSINESSUNIT = {
'lg parts': 'Large Parts',
'lg part': 'Large Parts',
'large part': 'Large Parts',
'sm spools': 'Small Spools',
'sm spool': 'Small Spools',
'large spoo': 'Large Spools',
'lg spools': 'Large Spools',
'lg spool': 'Large Spools',
'hptcr': 'HPT Rotor',
'bliskcr': 'Blisk Rotor',
'blisk': 'Blisk Rotor',
'leap x': 'LEAP-X',
}
# Raw export location string (lowercased, stripped) -> exact locationname in DB.
# Only for codes that do NOT resolve by dropping the WM- prefix. Leave a value
# as None to say "known to be unmappable, do not touch location for this row".
#
# The entries below the two None rows were promoted from the dry-run's
# high-confidence suggestions and verified to exist in the locations table. A
# slash-joined bay code (T6/U6) names two adjacent bays, but an asset carries a
# single locationid, so each maps to the FIRST (primary) bay - the deterministic
# choice a human confirmed. LD7W and LD6W are intentionally absent: they had no
# existing match and need a location created before they can be aliased.
LOCATION_OVERRIDES = {
'wm': None,
'wm-toolroom': None,
'wm-t6/u6': 'T6',
'wm-t9/10': 'T9',
'wm-u9/10': 'U9',
'wm-t4/u4': 'T4',
'wm-ld9w(ge90)': 'LD9W',
'wm-m6.3': 'M6',
'wm-l1-4': 'L1',
'wm-ld7w': None,
'wm-ld6w': None,
}

91
scripts/site_imports/winc/batches.py Normal file
View File

@@ -0,0 +1,91 @@
"""WINC machine adoption - BATCHES (list applied/undone import batches).
A read-only companion to writer.py and undo.py: it prints the WINC batches
newest-first so an operator can find a batchid to feed `--batch` into the undo
CLI. Like the other WINC commands it is WINC-named only for discoverability; the
underlying importbatches table is import-agnostic, so this simply filters to the
winc-machine-adoption kind.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
from pathlib import Path
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from sqlalchemy import text
from scripts.site_imports.winc.dry_run import build_app_connection
_KIND = 'winc-machine-adoption'
def _iso(value):
# PyMySQL hands back datetime objects; SQLite hands back the stored ISO
# string. Normalise both to an ISO string so the output is stable.
if value is None:
return '-'
if hasattr(value, 'isoformat'):
return value.isoformat()
return str(value)
def list_batches(connection, limit):
rows = connection.execute(text(
'SELECT batchid, kind, rowcount, createdat, createdby, status, '
'undoneat FROM importbatches WHERE kind = :kind '
'ORDER BY createdat DESC LIMIT :limit'),
{'kind': _KIND, 'limit': limit}).fetchall()
return [
{'batchid': r[0], 'kind': r[1], 'rowcount': r[2],
'createdat': _iso(r[3]), 'createdby': r[4], 'status': r[5],
'undoneat': _iso(r[6])}
for r in rows
]
def _print_table(batches):
if not batches:
print('No WINC import batches found.')
return
header = '%-36s %-8s %5s %-20s %-16s %s' % (
'batchid', 'status', 'rows', 'createdat', 'createdby', 'undoneat')
print(header)
print('-' * len(header))
for batch in batches:
print('%-36s %-8s %5s %-20s %-16s %s' % (
batch['batchid'], batch['status'], batch['rowcount'],
batch['createdat'], batch['createdby'] or '-', batch['undoneat']))
def main(argv=None):
parser = argparse.ArgumentParser(
description='List applied/undone WINC import batches, newest first.')
parser.add_argument('--limit', type=int, default=50,
help='Maximum number of batches to show (default 50).')
parser.add_argument('--json', action='store_true',
help='Emit the batches as JSON instead of a table.')
args = parser.parse_args(argv)
if args.limit < 1:
parser.error('--limit must be a positive integer')
_app, connection = build_app_connection()
batches = list_batches(connection, args.limit)
if args.json:
import json
print(json.dumps(batches, indent=2))
else:
_print_table(batches)
return 0
if __name__ == '__main__':
sys.exit(main())

385
scripts/site_imports/winc/dry_run.py Normal file
View File

@@ -0,0 +1,385 @@
"""Dry-run analyzer for the WINC (West Manufacturing) machine adoption.
READ ONLY. This script opens the database, reads the May asset export, decides
what an import WOULD change, and prints a report. It never writes. Nothing here
commits, updates, or inserts - that is a separate tool, written only once this
report reads clean and a human has signed off on it.
What it does, in order:
1. Load every existing machine's asset number and build the crosswalk key
(the workstation number, e.g. 0083A) the export will be matched on.
2. Read the export with a real CSV parser (quoted commas, BOM) and, for each
row that carries a workstation number, decide:
- does it match an existing machine? (in scope vs skipped)
- what name would we set? (export description)
- what location would we set? (WM-<bay> -> locations, via
aliases.LOCATION_OVERRIDES for
the exceptions)
- what business unit would we set? (eq1 -> aliases.EQ1_TO_
BUSINESSUNIT -> businessunits)
3. Classify each field resolution as RESOLVED, BLANK (nothing to set, leave
as-is), NEEDS-CREATE (target value is valid but not yet in the database),
or UNRESOLVED (no alias entry - a human must add one to aliases.py).
4. Print a summary and, with --rows, a per-row table. With --csv PATH it also
writes the full plan so the numbers can be diffed between runs.
Run it against the real database read-only:
python -m scripts.site_imports.winc.dry_run --export "exports/ASSET_current May26.csv"
python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --rows
python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --csv plan.csv
The design goal is reliability over cleverness: one match rule, one location
rule, one business-unit rule, every exception in aliases.py, and a report that
tells a human exactly what is unresolved so the alias table can be completed
before anyone runs the writer.
"""
import argparse
import csv
import os
import re
import sys
from collections import OrderedDict
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
# Same reasoning as scripts/export_catalog.py: a plain `python -m` run gets
# neither wsgi.py's nor the flask CLI's .env load, so DATABASE_URL would be
# unset and config would silently fall back to a hardcoded localhost URL. Load
# it here before anything reads config.
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from scripts.site_imports.approval import build_packet, format_packet
from scripts.site_imports.suggest import suggest
from scripts.site_imports.resolver import (
BLANK,
NEEDS_CREATE,
RESOLVED,
Resolution,
UNRESOLVED,
load_lookup,
norm as _norm,
resolve_alias,
resolve_name,
)
from scripts.site_imports.winc.aliases import (
EQ1_TO_BUSINESSUNIT,
LOCATION_OVERRIDES,
)
# WM- is the only bay prefix in the WINC export; a location code resolves by
# dropping it and matching a location by name. Everything else is an override.
_WM_PREFIX = re.compile(r'^WM-?', re.IGNORECASE)
# A workstation number is a LEADING run of digits with an optional single letter
# suffix. The export writes it clean (W0083A); the machines table stores the
# same number but often with a trailing descriptor (0083 Debur, 0083A Inspect).
# So the key is the leading token on both sides, matched unanchored - anchoring
# with $ would drop every existing machine whose name carries a descriptor and
# silently shrink the in-scope set. The leading W is the single biggest gotcha
# in this data, so it is stripped here in exactly one place.
_WSNUM = re.compile(r'^(\d{3,5}[A-Z]?)')
def workstation_key(assetnum):
"""The crosswalk key (e.g. '0083A') for an asset number, or None.
Strips a leading W and keeps the leading run of digits plus an optional
single trailing letter, ignoring any descriptor that follows. Returns None
for anything that is not a workstation number (blank rows, facilities codes,
named test machines) so non-machine rows fall out of scope on their own
rather than being force-fit.
"""
text = (assetnum or '').strip().upper()
if text.startswith('W'):
text = text[1:]
match = _WSNUM.match(text)
return match.group(1) if match else None
def load_existing_machine_keys(connection):
"""Map workstation key -> list of existing machine asset numbers.
A list, not a single value, because the machines table has a few numbers
that collide once reduced to their workstation key (0083 exists as
'0083 Debur' and '0083 Inspection'). The analyzer reports those as ambiguous
rather than guessing which one the export row means.
"""
from sqlalchemy import text
keys = OrderedDict()
rows = connection.execute(text(
"SELECT a.assetnumber "
"FROM assets a JOIN machines m ON m.assetid = a.assetid"))
for (assetnumber,) in rows:
key = workstation_key(assetnumber)
if key:
keys.setdefault(key, []).append(assetnumber)
return keys
def load_locations(connection, display=None):
"""locationname (normalized) -> locationid, from the live locations table."""
return load_lookup(connection, 'locations', 'locationid', 'locationname',
display=display)
def load_businessunits(connection):
"""businessunit (normalized) -> businessunitid, from the live table."""
return load_lookup(connection, 'businessunits', 'businessunitid',
'businessunit')
def resolve_location(raw_location, locations_by_name):
"""Resolution for a raw WM bay code.
Rule: an override in aliases wins; otherwise drop the WM- prefix and match a
location by name. A blank export cell means "leave the asset's location as
is" and is reported as BLANK, not UNRESOLVED, because there is nothing to
set and nothing for a human to fix. Overrides are checked first because they
carry the exceptions the plain prefix rule cannot handle.
"""
raw = (raw_location or '').strip()
if not raw:
return Resolution(BLANK, None, None)
if _norm(raw) in LOCATION_OVERRIDES:
return resolve_alias(raw, LOCATION_OVERRIDES, locations_by_name)
name = _WM_PREFIX.sub('', raw).strip()
return resolve_name(name, locations_by_name)
def resolve_businessunit(raw_eq1, businessunits_by_name):
"""Resolution for an eq1 value stream.
A blank eq1 is BLANK (leave the asset alone). A non-blank value must have an
alias entry; without one it is UNRESOLVED so a human adds it to aliases.py. A
mapped canonical name that does not yet exist in businessunits is
NEEDS-CREATE - flagged, never created here.
"""
return resolve_alias(raw_eq1, EQ1_TO_BUSINESSUNIT, businessunits_by_name)
def read_export(path):
"""Rows of the export as dicts. utf-8-sig handles the BOM; the csv module
handles the quoted commas embedded in some descriptions."""
with open(path, newline='', encoding='utf-8-sig') as handle:
return list(csv.DictReader(handle))
def parse_export_text(text):
"""Rows of an export given its raw CSV text (same shape as read_export).
Split out so an API route can hand the analyzer an uploaded file's bytes
without writing them to disk, while the CLI keeps reading from a path.
"""
import io
return list(csv.DictReader(io.StringIO(text)))
def analyze(connection, export_path):
"""Build the per-row plan from a file path. Returns (plan, summary, lookups)."""
return analyze_rows(connection, read_export(export_path))
def analyze_rows(connection, export_rows):
"""Build the per-row plan. Returns (plan_rows, summary_dict, lookups).
Takes already-parsed export rows so the same analysis serves both the CLI
(which reads a path) and the API route (which reads uploaded bytes).
"""
existing = load_existing_machine_keys(connection)
location_names = {}
locations = load_locations(connection, display=location_names)
businessunits = load_businessunits(connection)
plan = []
summary = OrderedDict([
('export_rows', len(export_rows)),
('non_workstation_rows', 0),
('in_scope', 0),
('skipped_no_existing_machine', 0),
('ambiguous_match', 0),
('location_' + RESOLVED, 0),
('location_' + BLANK, 0),
('location_' + NEEDS_CREATE, 0),
('location_' + UNRESOLVED, 0),
('businessunit_' + RESOLVED, 0),
('businessunit_' + BLANK, 0),
('businessunit_' + NEEDS_CREATE, 0),
('businessunit_' + UNRESOLVED, 0),
])
for row in export_rows:
assetnum = (row.get('assetnum') or '').strip()
key = workstation_key(assetnum)
if key is None:
summary['non_workstation_rows'] += 1
continue
matches = existing.get(key, [])
if not matches:
summary['skipped_no_existing_machine'] += 1
continue
summary['in_scope'] += 1
ambiguous = len(matches) > 1
if ambiguous:
summary['ambiguous_match'] += 1
loc = resolve_location(row.get('location'), locations)
bu = resolve_businessunit(row.get('eq1'), businessunits)
summary['location_' + loc.status] += 1
summary['businessunit_' + bu.status] += 1
plan.append(OrderedDict([
('assetnum', assetnum),
('workstation_key', key),
('matched_asset', '|'.join(matches)),
('ambiguous', 'yes' if ambiguous else ''),
('name', (row.get('description') or '').strip()),
('raw_location', (row.get('location') or '').strip()),
('location_status', loc.status),
('locationid', loc.id if loc.id is not None else ''),
('location_name', loc.name or ''),
('raw_eq1', (row.get('eq1') or '').strip()),
('businessunit_status', bu.status),
('businessunitid', bu.id if bu.id is not None else ''),
('businessunit_name', bu.name or ''),
]))
return plan, summary, {'locations': locations,
'location_names': location_names,
'businessunits': businessunits}
def print_summary(summary):
print('=' * 60)
print('WINC machine adoption - DRY RUN (no changes written)')
print('=' * 60)
for label, value in summary.items():
print(' %-32s %6d' % (label.replace('_', ' '), value))
def print_unresolved(plan, lookups=None):
loc = OrderedDict()
bu = OrderedDict()
for row in plan:
if row['location_status'] == UNRESOLVED:
loc[row['raw_location']] = loc.get(row['raw_location'], 0) + 1
if row['businessunit_status'] == UNRESOLVED:
bu[row['raw_eq1']] = bu.get(row['raw_eq1'], 0) + 1
needs_loc = OrderedDict()
needs_bu = OrderedDict()
for row in plan:
if row['location_status'] == NEEDS_CREATE:
needs_loc[row['location_name']] = needs_loc.get(row['location_name'], 0) + 1
if row['businessunit_status'] == NEEDS_CREATE:
needs_bu[row['businessunit_name']] = needs_bu.get(row['businessunit_name'], 0) + 1
if loc:
print('\nUNRESOLVED locations (add to aliases.LOCATION_OVERRIDES):')
location_lookup = (lookups or {}).get('locations') or {}
location_names = (lookups or {}).get('location_names') or {}
for value, count in sorted(loc.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
stripped = _WM_PREFIX.sub('', value).strip()
for hint in suggest(value, stripped, location_lookup,
display=location_names):
print(' -> suggest "%s" (%s, confidence %s)'
% (hint.candidate, hint.reason, hint.confidence))
if bu:
print('\nUNRESOLVED eq1 (add to aliases.EQ1_TO_BUSINESSUNIT):')
for value, count in sorted(bu.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
if needs_loc:
print('\nNEEDS-CREATE locations (mapped, but not yet in locations):')
for value, count in sorted(needs_loc.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
if needs_bu:
print('\nNEEDS-CREATE business units (mapped, but not yet in businessunits):')
for value, count in sorted(needs_bu.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
def print_rows(plan):
print('\nPER-ROW PLAN:')
header = ('assetnum', 'matched_asset', 'name', 'raw_location',
'location_status', 'raw_eq1', 'businessunit_status')
print(' ' + ' | '.join(header))
for row in plan:
print(' ' + ' | '.join(str(row[h]) for h in header))
def write_csv(plan, path):
if not plan:
return
with open(path, 'w', newline='', encoding='utf-8') as handle:
writer = csv.DictWriter(handle, fieldnames=list(plan[0].keys()))
writer.writeheader()
writer.writerows(plan)
def build_app_connection():
"""Yield a live SQLAlchemy connection inside an app context.
Returns (app, connection). Caller is responsible for closing nothing - the
connection lives for the app context and the process exits right after.
"""
from shopdb import create_app
from shopdb.api import db
app = create_app()
context = app.app_context()
context.push()
return app, db.session.connection()
def main(argv=None):
parser = argparse.ArgumentParser(
description='Dry-run analyzer for WINC machine adoption. Read only.')
parser.add_argument('--export', required=True,
help='Path to the ASSET export CSV.')
parser.add_argument('--rows', action='store_true',
help='Print the full per-row plan.')
parser.add_argument('--csv', dest='csv_out',
help='Write the full per-row plan to this CSV path.')
parser.add_argument('--packet', action='store_true',
help='Print the approval packet (hash, needs-create '
'and blocking summary) a writer would enforce.')
args = parser.parse_args(argv)
if not os.path.exists(args.export):
parser.error('export file not found: %s' % args.export)
_app, connection = build_app_connection()
plan, summary, lookups = analyze(connection, args.export)
print_summary(summary)
print_unresolved(plan, lookups)
if args.rows:
print_rows(plan)
if args.csv_out:
write_csv(plan, args.csv_out)
print('\nWrote per-row plan: %s (%d rows)' % (args.csv_out, len(plan)))
if args.packet:
packet = build_packet(
'winc-machine-adoption', plan, summary,
[('businessunit', 'businessunit_status', 'raw_eq1',
'businessunit_name'),
('location', 'location_status', 'raw_location', 'location_name')])
print('\n' + format_packet(packet))
return 0
if __name__ == '__main__':
sys.exit(main())

55
scripts/site_imports/winc/undo.py Normal file
View File

@@ -0,0 +1,55 @@
"""WINC machine adoption - UNDO (reverse an applied import batch).
A thin CLI over the shared, import-agnostic journal.undo_batch. It is WINC-named
only for discoverability; the actual work is generic, so the same reversal serves
any importer that wrote through the journal. Reverting is conservative: a cell a
human changed since the import is left as-is and reported, and a created business
unit is deleted only when nothing outside the batch still references it.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
from pathlib import Path
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from scripts.site_imports import journal
from scripts.site_imports.winc.dry_run import build_app_connection
from scripts.site_imports.winc.writer import _UNDO_REFERENCES
def main(argv=None):
parser = argparse.ArgumentParser(
description='Undo an applied WINC import batch (reversible journal).')
parser.add_argument('--batch', required=True,
help='The batch id to reverse.')
parser.add_argument('--commit', action='store_true',
help='Actually reverse. Without it, this is a no-op.')
args = parser.parse_args(argv)
if not args.commit:
print('DRY: pass --commit to reverse batch %s' % args.batch)
return 0
_app, connection = build_app_connection()
from shopdb.api import db
try:
result = journal.undo_batch(connection, args.batch,
references=_UNDO_REFERENCES)
db.session.commit()
except Exception:
db.session.rollback()
raise
print(journal.format_undo_result(result))
return 0
if __name__ == '__main__':
sys.exit(main())

278
scripts/site_imports/winc/writer.py Normal file
View File

@@ -0,0 +1,278 @@
"""WINC machine adoption - WRITER (applies the approved plan, reversibly).
This is the write half of the WINC import; scripts/site_imports/winc/dry_run.py
is the read-only half. The writer never invents its own view of the data: it
rebuilds the exact same plan the dry-run produced, verifies that plan against the
approval packet the human signed off on, and only then writes - recording every
change to the shared import journal so the whole batch can be undone.
The contract, in order (the one documented in approval.py, made concrete here):
1. rebuild the plan from the same export + live DB,
2. build a packet from it,
3. compare planhash to the approved packet - abort on mismatch (drift),
4. refuse if the packet is blocking (any unresolved value),
5. require explicit acknowledgment for every needs-create group,
6. write inside ONE transaction, stamping the batch id onto the journal:
a. create each acknowledged needs-create business unit (journal insert),
b. update each in-scope asset's name / location / business unit
(journal update, capturing the pre-image first).
Everything commits together or not at all. Ambiguous rows (one workstation key
matching several machines) are never written - the dry-run flags them and the
writer skips them, because guessing which machine an export row means is exactly
the kind of cleverness this import refuses.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
from pathlib import Path
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from scripts.site_imports import journal
from scripts.site_imports.approval import build_packet, packet_matches_plan
from scripts.site_imports.resolver import NEEDS_CREATE, RESOLVED, norm
from scripts.site_imports.winc.dry_run import (
analyze,
build_app_connection,
)
# The packet groups the WINC dry-run declares, reused verbatim so the writer
# hashes and reads the plan exactly the way approval did.
_PACKET_GROUPS = [
('businessunit', 'businessunit_status', 'raw_eq1', 'businessunit_name'),
('location', 'location_status', 'raw_location', 'location_name'),
]
# References into a created row's id, so undo can tell whether an inserted
# business unit is still pointed at by something outside this batch before it
# hard-deletes it. Shared with the undo CLI and the web undo path.
_UNDO_REFERENCES = {
'businessunits': [('assets', 'businessunitid')],
}
# References into a created row's id, so undo can tell whether an inserted
# business unit is still pointed at by something outside this batch before it
# hard-deletes it.
# Max width of assets.name; the export description can exceed it (a few rows
# carry a whole safety notice in the field), so a set name is clamped rather
# than letting the whole transaction fail on one over-long row. The pre-image is
# journalled, so the clamp is reversible like any other change.
_ASSET_NAME_MAXLEN = 100
class WriterError(Exception):
"""The plan cannot be written as approved (drift, blocking, or missing
acknowledgment). Carries a human-readable reason; never a partial write."""
def _load_asset_ids(connection):
"""assetnumber -> assetid for every asset, so a plan row (which carries the
matched asset *number*) can be turned into the id the update targets."""
from sqlalchemy import text
out = {}
for assetid, assetnumber in connection.execute(text(
'SELECT assetid, assetnumber FROM assets')):
out[assetnumber] = assetid
return out
def _create_businessunit(connection, batchid, sequence, name):
"""Insert one business unit and journal it. Returns (new_id, next_sequence).
Only ever called for an acknowledged needs-create name, so the journal's
insert entry describes strictly a row this batch created."""
from sqlalchemy import text
now = journal._utcnow()
result = connection.execute(text(
'INSERT INTO businessunits '
'(businessunit, createddate, modifieddate, isactive) '
'VALUES (:name, :now, :now, 1)'),
{'name': name, 'now': now})
new_id = result.lastrowid
journal.record_insert(connection, batchid, sequence, 'businessunits',
new_id)
return new_id, sequence + 1
def _update_cell(connection, batchid, sequence, table, targetid, column,
oldvalue, newvalue):
"""Apply one cell update and journal its pre/post image. Returns the next
sequence. Skips (and does not journal) a no-op where the value is unchanged."""
from sqlalchemy import text
if journal._as_text(oldvalue) == journal._as_text(newvalue):
return sequence
connection.execute(text(
'UPDATE %s SET %s = :newvalue WHERE %s = :targetid'
% (table, column, _asset_pk(table))),
{'newvalue': newvalue, 'targetid': targetid})
journal.record_update(connection, batchid, sequence, table, targetid,
column, oldvalue, newvalue)
return sequence + 1
def _asset_pk(table):
# The writer only updates the assets table; keep the mapping explicit rather
# than reflecting, so an unexpected table name fails loudly.
return {'assets': 'assetid'}[table]
def apply_plan(connection, plan, approved_packet, acknowledge=None,
createdby=None):
"""Apply an approved WINC plan, reversibly. Returns the batch id.
connection a live SQLAlchemy connection in an open transaction.
plan the per-row plan from dry_run.analyze (rebuilt fresh).
approved_packet the packet a human signed off on.
acknowledge set of needs-create group names the human confirmed
(e.g. {'businessunit'}); required for every group the
packet lists under needscreate.
createdby who acknowledged, stamped on the batch.
Raises WriterError before touching anything if the plan drifted, is
blocking, or a needs-create group was not acknowledged.
"""
acknowledge = set(acknowledge or ())
# (2)-(3) rebuild the packet and prove nothing drifted since approval.
current_packet = build_packet('winc-machine-adoption', plan,
approved_packet.get('counts', {}),
_PACKET_GROUPS)
if not packet_matches_plan(approved_packet, plan):
raise WriterError(
'plan has changed since approval (hash mismatch) - re-run the '
'dry-run and re-approve before importing')
# (4) never write over an unresolved value.
if current_packet.get('blocking'):
raise WriterError(
'plan is blocking: unresolved values remain - add the missing '
'aliases and re-run the dry-run')
# (5) every needs-create group must be explicitly acknowledged.
missing_ack = set(current_packet.get('needscreate', {})) - acknowledge
if missing_ack:
raise WriterError(
'needs-create not acknowledged for: %s - confirm these will be '
'created before importing' % ', '.join(sorted(missing_ack)))
batchid = approved_packet['batchid']
journal.open_batch(connection, approved_packet, createdby=createdby)
sequence = 1
# (6a) create acknowledged needs-create business units once each, first, so
# the asset updates below can point at them.
created_bu = {}
for row in plan:
if row['businessunit_status'] != NEEDS_CREATE:
continue
name = row['businessunit_name']
key = norm(name)
if key not in created_bu:
created_bu[key], sequence = _create_businessunit(
connection, batchid, sequence, name)
# (6b) update each in-scope, unambiguous asset.
asset_ids = _load_asset_ids(connection)
from sqlalchemy import text
for row in plan:
if row['ambiguous']:
continue
assetnumber = row['matched_asset']
assetid = asset_ids.get(assetnumber)
if assetid is None:
continue
current = connection.execute(text(
'SELECT name, locationid, businessunitid FROM assets '
'WHERE assetid = :assetid'), {'assetid': assetid}).fetchone()
cur_name, cur_location, cur_bu = current
if row['name']:
new_name = row['name'][:_ASSET_NAME_MAXLEN]
sequence = _update_cell(connection, batchid, sequence, 'assets',
assetid, 'name', cur_name, new_name)
if row['location_status'] == RESOLVED and row['locationid'] != '':
sequence = _update_cell(connection, batchid, sequence, 'assets',
assetid, 'locationid', cur_location,
row['locationid'])
new_bu = None
if row['businessunit_status'] == RESOLVED and row['businessunitid'] != '':
new_bu = row['businessunitid']
elif row['businessunit_status'] == NEEDS_CREATE:
new_bu = created_bu[norm(row['businessunit_name'])]
if new_bu is not None:
sequence = _update_cell(connection, batchid, sequence, 'assets',
assetid, 'businessunitid', cur_bu, new_bu)
return batchid
def main(argv=None):
parser = argparse.ArgumentParser(
description='WINC machine adoption writer. Applies an approved plan '
'reversibly. Requires --commit to write.')
parser.add_argument('--export', required=True,
help='Path to the ASSET export CSV.')
parser.add_argument('--acknowledge', action='append', default=[],
metavar='GROUP',
help='Acknowledge a needs-create group (repeatable), '
'e.g. --acknowledge businessunit.')
parser.add_argument('--createdby', default=None,
help='Who is approving this import (stamped on batch).')
parser.add_argument('--commit', action='store_true',
help='Actually write. Without it, this is a no-op that '
'reports what would happen.')
args = parser.parse_args(argv)
if not os.path.exists(args.export):
parser.error('export file not found: %s' % args.export)
_app, connection = build_app_connection()
plan, summary, _lookups = analyze(connection, args.export)
packet = build_packet('winc-machine-adoption', plan, summary,
_PACKET_GROUPS)
if not args.commit:
print('DRY: rebuilt plan of %d rows, hash %s' % (
packet['rowcount'], packet['planhash']))
print('DRY: pass --commit with --acknowledge for each needs-create '
'group to write. Needs-create groups: %s'
% (', '.join(packet.get('needscreate', {})) or 'none'))
return 0
from shopdb.api import db
try:
batchid = apply_plan(connection, plan, packet,
acknowledge=args.acknowledge,
createdby=args.createdby)
db.session.commit()
except WriterError as error:
db.session.rollback()
print('REFUSED: %s' % error)
return 2
except Exception:
db.session.rollback()
raise
print('COMMITTED batch %s' % batchid)
print(' to undo: python -m scripts.site_imports.winc.undo --batch %s'
% batchid)
return 0
if __name__ == '__main__':
sys.exit(main())

8
shopdb/__init__.py
View File

@@ -104,7 +104,13 @@ from .plugins import plugin_manager
# defeated by rotating the header. The helper resolves from remote_addr, which
# ProxyFix has already filled from the header when TRUSTED_PROXY_COUNT says a
# proxy is really in front. Additive names on the surface, minor bump.
__contract_version__ = '0.29.0'
# 0.30.0: added ImportBatch, ImportJournalEntry, and the BATCH_*/OP_* constants
# to shopdb.api so a plugin's bulk-import UI can list applied import batches and
# reverse one through the shared reversible journal without importing core model
# internals. The journal records the primitive mutations a bulk import makes
# (update a cell, insert a row) tied to a batch id, and a generic undo replays
# them in reverse. Additive names on the surface, minor bump.
__contract_version__ = '0.30.0'
# Product release version (see ADR-007). The product version and the
# plugin-contract version above are distinct series with independent

16
shopdb/api/__init__.py
View File

@@ -140,6 +140,22 @@ from shopdb.utils.mailer import send_email, send_alert, send_webhook
from shopdb.utils.cmmc_usb_db import cmmc_usb_connection
# Reversible bulk-import journal. A plugin that drives a bulk import (a site
# asset adoption, an integration sync) records the primitive changes it makes so
# the whole batch can be undone. ImportBatch is the identity approval, write, and
# undo all reference; ImportJournalEntry is one reversible mutation. Exposed so a
# plugin's import UI can list applied batches and reverse one without reaching
# into core internals. See the PLUGIN-HOOKS wiki page.
from shopdb.core.models import (
ImportBatch,
ImportJournalEntry,
BATCH_APPLIED,
BATCH_UNDONE,
OP_UPDATE,
OP_INSERT,
)
def audit_log(
action: str,
entitytype: str,

128
shopdb/core/api/settings.py
View File

@@ -595,6 +595,134 @@ def import_asset_csvs():
})
def _winc_upload_text():
"""Read the single uploaded WINC export as text, or (None, error_response)."""
uploads = request.files.getlist('files')
if not uploads:
return None, error_response(ErrorCodes.VALIDATION_ERROR,
'No export file was uploaded')
upload = uploads[0]
name = upload.filename or ''
if not name.lower().endswith('.csv'):
return None, error_response(ErrorCodes.VALIDATION_ERROR,
'The WINC export must be a .csv file')
try:
return upload.read().decode('utf-8-sig'), None
except (UnicodeDecodeError, OSError):
return None, error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
@settings_bp.route('/asset-import/winc/preview', methods=['POST'])
@require_role('admin')
def winc_import_preview():
"""Read-only WINC analysis: report what a commit would do. Writes nothing.
WINC (a Maximo-style asset dump) is not a canonical-CSV source: it updates
existing machines only, needs acknowledgment before creating business units,
and is reversible. This returns the approval packet the UI renders - counts,
the business units it would create, and any unresolved values that block it.
"""
from shopdb.core.services.wincimport import preview, WincImportError
export_text, failure = _winc_upload_text()
if failure is not None:
return failure
try:
report = preview(export_text)
except WincImportError as exc:
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
finally:
# A preview must leave nothing behind even though it only read: the
# analyzer runs against the request-scoped session.
db.session.rollback()
return success_response(report)
@settings_bp.route('/asset-import/winc/commit', methods=['POST'])
@require_role('admin')
def winc_import_commit():
"""Apply a WINC plan reversibly. Requires acknowledgment for needs-create.
The 'acknowledge' form field is a comma-separated list of needs-create group
names the admin confirmed (e.g. 'businessunit'). The whole write is one
transaction recorded to the import journal, so it can be undone later.
"""
from shopdb.core.services.wincimport import commit as winc_commit
from shopdb.core.services.wincimport import WincImportError
export_text, failure = _winc_upload_text()
if failure is not None:
return failure
acknowledge = [
part.strip() for part in
(request.form.get('acknowledge') or '').split(',') if part.strip()]
# The hash the PREVIEW returned. Required here, because without it the
# writer's drift check compares a freshly built packet against the plan it
# was built from and can never fail - so "the admin approved this plan"
# meant only "the admin ticked these group names".
planhash = (request.form.get('planhash') or '').strip()
if not planhash:
return error_response(
ErrorCodes.VALIDATION_ERROR,
'planhash is required: send the value the preview returned, so the '
'commit can prove it is applying the plan that was approved.')
try:
report = winc_commit(export_text, acknowledge=acknowledge,
createdby=get_jwt_identity(), planhash=planhash)
AuditLog.log(
'imported', 'SiteAssets', entityname='winc-machine-adoption',
changes={'batchid': report.get('batchid'),
'rows': report.get('rowcount')})
db.session.commit()
except WincImportError as exc:
db.session.rollback()
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
except Exception:
db.session.rollback()
raise
return success_response(report)
@settings_bp.route('/asset-import/winc/batches', methods=['GET'])
@require_role('admin')
def winc_import_batches():
"""List applied/undone WINC import batches, newest first (for the undo UI)."""
from shopdb.core.services.wincimport import list_batches
return success_response({'batches': list_batches()})
@settings_bp.route('/asset-import/winc/undo', methods=['POST'])
@require_role('admin')
def winc_import_undo():
"""Reverse an applied WINC import batch by id. Conservative and reversible."""
from shopdb.core.services.wincimport import undo as winc_undo
payload = request.get_json(silent=True) or {}
batchid = (payload.get('batchid') or request.form.get('batchid') or '').strip()
if not batchid:
return error_response(ErrorCodes.VALIDATION_ERROR, 'A batchid is required')
try:
result = winc_undo(batchid)
AuditLog.log(
'undone', 'SiteAssets', entityname='winc-machine-adoption',
changes={'batchid': batchid, 'reverted': result['reverted'],
'conflicts': len(result['conflicts'])})
db.session.commit()
except ValueError as exc:
db.session.rollback()
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
except Exception:
db.session.rollback()
raise
return success_response(result)
@settings_bp.route('', methods=['GET'])
@jwt_required(optional=True)
def list_settings():

9
shopdb/core/models/__init__.py
View File

@@ -23,6 +23,8 @@ from .setting import Setting
from .auditlog import AuditLog
from .customfield import CustomField, CustomFieldValue
from .apitoken import ApiToken
from .importbatch import (ImportBatch, ImportJournalEntry, BATCH_APPLIED,
BATCH_UNDONE, OP_UPDATE, OP_INSERT)
__all__ = [
# Base
@@ -82,4 +84,11 @@ __all__ = [
'CustomFieldValue',
# Personal API tokens
'ApiToken',
# Import batch + journal (reversible bulk imports)
'ImportBatch',
'ImportJournalEntry',
'BATCH_APPLIED',
'BATCH_UNDONE',
'OP_UPDATE',
'OP_INSERT',
]

86
shopdb/core/models/importbatch.py Normal file
View File

@@ -0,0 +1,86 @@
"""Import batch + journal models (reversible bulk imports).
PLATFORM-LEVEL, IMPORT-AGNOSTIC. These two tables let any bulk-import writer
record exactly what it changed so the change can be reversed later. Nothing here
knows about Maximo, WINC, machines, or any one source: an importer records
primitive mutations (set column X on row Y, or insert row Y) tied to a batch id,
and a generic undo replays them in reverse.
importbatches one row per approved write - the identity that approval, the
write, and the eventual undo all reference (the same value
the approval packet calls batchid).
importjournalentries one row per primitive mutation the writer made, ordered so
undo can reverse them (inserts removed after the updates that
pointed at them are reverted).
The undo contract, kept deliberately conservative:
- update is reverted only if the cell still holds the value we wrote; a value a
human changed since is left as-is and reported (never clobbered).
- insert is hard-deleted, but ONLY the row this batch created and only when
nothing outside the batch still references it.
"""
from shopdb.extensions import db
# Batch lifecycle.
BATCH_APPLIED = 'applied'
BATCH_UNDONE = 'undone'
# Journal operation kinds.
OP_UPDATE = 'update'
OP_INSERT = 'insert'
class ImportBatch(db.Model):
"""One approved import write. The batchid is the approval packet's batchid,
carried through write and undo so all three reference one identity."""
__tablename__ = 'importbatches'
batchid = db.Column(db.String(36), primary_key=True)
kind = db.Column(db.String(64), nullable=False)
planhash = db.Column(db.String(64), nullable=False)
rowcount = db.Column(db.Integer, nullable=False, default=0)
createdat = db.Column(db.DateTime, nullable=False)
createdby = db.Column(db.String(100), nullable=True)
status = db.Column(db.String(16), nullable=False, default=BATCH_APPLIED)
undoneat = db.Column(db.DateTime, nullable=True)
entries = db.relationship(
'ImportJournalEntry',
backref='batch',
cascade='all, delete-orphan',
order_by='ImportJournalEntry.sequence',
)
def __repr__(self):
return f"<ImportBatch {self.kind} {self.batchid} {self.status}>"
class ImportJournalEntry(db.Model):
"""One reversible primitive mutation made by an import writer."""
__tablename__ = 'importjournalentries'
# SQLite only autoincrements a plain INTEGER PRIMARY KEY, not BIGINT, so the
# test database needs the Integer variant while MySQL keeps BIGINT for range.
entryid = db.Column(
db.BigInteger().with_variant(db.Integer, 'sqlite'),
primary_key=True)
batchid = db.Column(
db.String(36),
db.ForeignKey('importbatches.batchid', ondelete='CASCADE'),
nullable=False,
index=True,
)
sequence = db.Column(db.Integer, nullable=False)
operation = db.Column(db.String(16), nullable=False)
targettable = db.Column(db.String(64), nullable=False)
targetid = db.Column(db.BigInteger, nullable=False)
columnname = db.Column(db.String(64), nullable=True)
oldvalue = db.Column(db.Text, nullable=True)
newvalue = db.Column(db.Text, nullable=True)
isreverted = db.Column(db.Boolean, nullable=False, default=False)
def __repr__(self):
return (f"<ImportJournalEntry {self.operation} "
f"{self.targettable}.{self.targetid}>")

154
shopdb/core/services/wincimport.py Normal file
View File

@@ -0,0 +1,154 @@
"""WINC machine-adoption import service (preview / commit / undo / list).
Web-facing wrapper over the validated WINC CLI helpers. The CLI in
scripts/site_imports/winc drives the same analyzer, writer, and journal; this
module adapts them to a Flask request: it works on the request-scoped
db.session connection (so the route owns the transaction and commit) and returns
plain JSON-serialisable dicts for the API.
WINC is NOT a source adapter like Azure/Thor. It never creates assets - it
updates existing machines only, requires acknowledgment before creating business
units, and records every change to the reversible import journal. That contract
lives here and in the writer, not in the generic canonical-CSV importer.
"""
from shopdb.extensions import db
from scripts.site_imports import journal
from scripts.site_imports.approval import build_packet
from scripts.site_imports.winc import writer as winc_writer
from scripts.site_imports.winc.dry_run import analyze_rows, parse_export_text
class WincImportError(Exception):
"""An uploaded export could not be analyzed or applied as WINC."""
def _connection():
return db.session.connection()
def _plan_and_packet(export_text):
"""Parse an uploaded export, build the plan and the approval packet."""
rows = parse_export_text(export_text)
if not rows:
raise WincImportError('the uploaded file had no rows')
if 'assetnum' not in rows[0]:
raise WincImportError(
'this does not look like a WINC asset export (no assetnum column)')
plan, summary, _lookups = analyze_rows(_connection(), rows)
packet = build_packet('winc-machine-adoption', plan, summary,
winc_writer._PACKET_GROUPS)
return plan, summary, packet
def _report(summary, packet):
"""Shape the packet + summary into the JSON the UI renders."""
return {
'kind': packet['kind'],
'planhash': packet['planhash'],
'rowcount': packet['rowcount'],
'summary': dict(summary),
'needscreate': {group: dict(items)
for group, items in packet.get('needscreate', {}).items()},
'unresolved': {group: dict(items)
for group, items in packet.get('unresolved', {}).items()},
'blocking': packet.get('blocking', False),
}
def preview(export_text):
"""Read-only: analyze the export and return the report the UI shows.
Writes nothing. The whole point is to surface what a commit WOULD do -
counts, the business units it would create (needs acknowledgment), and any
unresolved values that block the import.
"""
_plan, summary, packet = _plan_and_packet(export_text)
report = _report(summary, packet)
report['committed'] = False
return report
def commit(export_text, acknowledge=None, createdby=None, planhash=None):
"""Apply the plan reversibly inside the request transaction.
Rebuilds the plan from the same upload, re-checks it against a freshly built
packet (drift, blocking, acknowledgment) and writes through the journal. The
caller owns the commit/rollback of db.session. Returns the report plus the
batchid on success.
planhash is the one the PREVIEW returned. It is what makes the approval mean
anything: the packet handed to the writer is built from this same upload, so
the writer's drift check compared a hash against itself and could never fail.
An admin previewing file A and committing file B - or the same file after the
locations or aliases moved underneath it - got whatever the second plan said,
gated only by the needs-create group NAMES they happened to tick.
Passing it is optional so the CLI path, which builds and applies in one go,
is unaffected. The web route requires it.
Raises WincImportError (mapped to a 400 by the route) when the plan is
blocking, a needs-create group was not acknowledged, or the plan no longer
matches the one that was approved.
"""
plan, summary, packet = _plan_and_packet(export_text)
if planhash is not None and packet['planhash'] != planhash:
raise WincImportError(
'This plan is not the one that was previewed - the upload or the '
'data behind it changed. Preview again and re-check what it would '
'create before committing.')
try:
batchid = winc_writer.apply_plan(
_connection(), plan, packet,
acknowledge=acknowledge, createdby=createdby)
except winc_writer.WriterError as exc:
raise WincImportError(str(exc))
report = _report(summary, packet)
report['committed'] = True
report['batchid'] = batchid
return report
def undo(batchid):
"""Reverse an applied batch. Caller owns the transaction. Returns a dict."""
result = journal.undo_batch(_connection(), batchid,
references=winc_writer._UNDO_REFERENCES)
return {
'batchid': result.batchid,
'reverted': result.reverted,
'conflicts': [
{'operation': conflict.operation, 'table': conflict.table,
'targetid': conflict.targetid, 'column': conflict.column,
'reason': conflict.reason}
for conflict in result.conflicts
],
}
def list_batches(limit=50):
"""Applied/undone WINC batches, newest first, for the UI's undo list."""
from sqlalchemy import text
def _iso(value):
# PyMySQL hands back datetime objects; SQLite (tests) hands back the
# stored ISO string. Normalise both to an ISO string with a Z suffix.
if value is None:
return None
if hasattr(value, 'isoformat'):
return value.isoformat() + 'Z'
return str(value)
rows = _connection().execute(text(
'SELECT batchid, kind, rowcount, createdat, createdby, status, '
'undoneat FROM importbatches WHERE kind = :kind '
'ORDER BY createdat DESC LIMIT :limit'),
{'kind': 'winc-machine-adoption', 'limit': limit}).fetchall()
return [
{'batchid': r[0], 'kind': r[1], 'rowcount': r[2],
'createdat': _iso(r[3]),
'createdby': r[4], 'status': r[5],
'undoneat': _iso(r[6])}
for r in rows
]

3
tests/test_plugin_migrations.py
View File

@@ -99,7 +99,8 @@ EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof'
# the operatorreports table for tray issue reports.
# 0010 renames the shipped business policy off the originating site's
# designator on installs that already stamped 0007 (ADR-015).
EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0010renamepolicy'
# 0011 adds the trayversion cache column to endpointstates.
EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0011trayversion'
# Plugins built after the cutover: their 0001 baseline really creates tables the
# core chain never owned.

57
tests/test_plugins/test_endpoints_agentstore.py Normal file
View File

@@ -0,0 +1,57 @@
"""Unit tests for the versioned agent binary store helpers.
Covers the pure, context-free helpers: version validation, PE magic and PE
file-version parsing. The staging/retention paths that touch the Flask instance
directory are exercised through the API tests; here we keep to the logic that
needs no app context.
The PE version parser walks the resource directory to the outer image's
RT_VERSION resource (not the first VS_FIXEDFILEINFO signature in the bytes),
because a self-contained single-file .NET exe bundles many framework assemblies
that each carry their own version block. Building a full PE with a resource
directory inline is impractical here, so the parser's correctness is verified
against real agent binaries in the API/integration layer; these unit tests pin
the surrounding validators and the malformed-input contract (never raises,
returns None).
"""
from plugins.endpoints.api import agentstore
def test_is_valid_version_accepts_dotted_numeric():
assert agentstore.is_valid_version('1')
assert agentstore.is_valid_version('1.6.3')
assert agentstore.is_valid_version('1.6.3.0')
def test_is_valid_version_rejects_bad_input():
assert not agentstore.is_valid_version('')
assert not agentstore.is_valid_version('v1.6')
assert not agentstore.is_valid_version('1.6.3.0.1')
assert not agentstore.is_valid_version('../etc')
assert not agentstore.is_valid_version('latest')
def test_is_valid_binary_key():
assert agentstore.is_valid_binary_key('agent')
assert agentstore.is_valid_binary_key('tray')
assert not agentstore.is_valid_binary_key('other')
def test_looks_like_pe():
assert agentstore.looks_like_pe(b'MZ\x90\x00')
assert not agentstore.looks_like_pe(b'PK\x03\x04')
assert not agentstore.looks_like_pe(b'')
def test_read_pe_file_version_none_on_non_pe():
assert agentstore.read_pe_file_version(b'not a pe file') is None
def test_read_pe_file_version_none_on_truncated_pe():
# MZ header but nothing else valid: must return None, never raise.
assert agentstore.read_pe_file_version(b'MZ' + b'\x00' * 200) is None
def test_read_pe_file_version_none_on_empty():
assert agentstore.read_pe_file_version(b'') is None

270
tests/test_winc_import_routes.py Normal file
View File

@@ -0,0 +1,270 @@
"""WINC machine-adoption import routes: preview / commit / undo.
WINC is not a canonical-CSV source adapter (that path is covered by
test_import_sources). It updates existing machines only, needs explicit
acknowledgment before creating a business unit, and every commit is journalled
so it can be undone. These tests drive the real routes end to end against the
in-memory schema: seed a machine asset, upload an export that matches it, and
assert what preview reports, what commit writes, and that undo puts it back.
"""
import io
import pytest
def _seed_machine(db, assetnumber='0083A Debur'):
"""Create one machine asset the export can match on, plus a J3 location.
Returns (assetid, locationid). The workstation key of '0083A Debur' is
'0083A', which the export row 'W0083A' reduces to, so the two crosswalk.
"""
from shopdb.core.models import AssetType, AssetStatus, Location
from shopdb.core.models.asset import Asset
from plugins.machines.models.machine import Machine
assettype = AssetType(assettype='machine')
status = AssetStatus(status='In Use')
location = Location(locationname='J3')
db.session.add_all([assettype, status, location])
db.session.flush()
asset = Asset(
assetnumber=assetnumber,
name='Old Name',
assettypeid=assettype.assettypeid,
statusid=status.statusid,
)
db.session.add(asset)
db.session.flush()
db.session.add(Machine(assetid=asset.assetid))
db.session.commit()
return asset.assetid, location.locationid
# An export whose single in-scope row matches the seeded machine. 'WM-J3'
# resolves to the J3 location; 'lg parts' aliases to the 'Large Parts' business
# unit (present or needs-create depending on the test).
EXPORT_CSV = (
'assetnum,description,location,eq1\r\n'
'W0083A,Renamed Machine,WM-J3,lg parts\r\n'
'FACILITIES-1,Not a machine,,\r\n'
)
def _upload(client, headers, url, csv_text=EXPORT_CSV, extra=None):
data = {'files': (io.BytesIO(csv_text.encode('utf-8')), 'winc.csv')}
if extra:
data.update(extra)
return client.post(url, data=data, content_type='multipart/form-data',
headers=headers)
def _commit(client, headers, csv_text=EXPORT_CSV, extra=None, planhash=None):
"""Preview, then commit the plan the preview returned.
This is the real flow, and the tests have to use it: the commit route now
requires the planhash from the preview. Committing without previewing was
what made the approval meaningless - the writer compared a freshly built
packet against the plan it was built from, so its drift check could not
fail and an admin could approve one plan and apply another.
Pass planhash explicitly to simulate approving one plan and committing a
different one.
"""
if planhash is None:
preview = _upload(client, headers, PREVIEW, csv_text=csv_text)
planhash = preview.get_json()['data']['planhash']
payload = dict(extra or {})
payload['planhash'] = planhash
return _upload(client, headers, COMMIT, csv_text=csv_text, extra=payload)
PREVIEW = '/api/settings/asset-import/winc/preview'
COMMIT = '/api/settings/asset-import/winc/commit'
UNDO = '/api/settings/asset-import/winc/undo'
BATCHES = '/api/settings/asset-import/winc/batches'
def test_preview_reports_plan_without_writing(client, db, auth_headers):
_seed_machine(db)
response = _upload(client, auth_headers, PREVIEW)
assert response.status_code == 200, response.get_json()
report = response.get_json()['data']
# One workstation row in scope, the facilities row filtered out.
assert report['committed'] is False
assert report['rowcount'] == 1
assert report['summary']['in_scope'] == 1
assert report['summary']['non_workstation_rows'] == 1
# 'Large Parts' does not exist yet, so it is flagged for acknowledgment.
assert report['needscreate']['businessunit'] == {'Large Parts': 1}
assert report['blocking'] is False
# A preview must not have touched the asset.
from shopdb.core.models.asset import Asset
assert Asset.query.first().name == 'Old Name'
def test_preview_rejects_non_winc_upload(client, db, auth_headers):
_seed_machine(db)
response = _upload(client, auth_headers, PREVIEW,
csv_text='foo,bar\r\n1,2\r\n')
assert response.status_code == 400
assert 'assetnum' in response.get_json()['data']['error']['message']
def test_commit_requires_acknowledgment_for_needs_create(client, db,
auth_headers):
_seed_machine(db)
# No acknowledge field: the needs-create business unit blocks the write.
response = _commit(client, auth_headers)
assert response.status_code == 400
message = response.get_json()['data']['error']['message'].lower()
assert 'acknowledge' in message
from shopdb.core.models.asset import Asset
from shopdb.core.models import BusinessUnit
assert Asset.query.first().name == 'Old Name'
assert BusinessUnit.query.count() == 0
def test_commit_applies_plan_and_creates_business_unit(client, db,
auth_headers):
assetid, locationid = _seed_machine(db)
response = _commit(client, auth_headers, extra={'acknowledge': 'businessunit'})
assert response.status_code == 200, response.get_json()
report = response.get_json()['data']
assert report['committed'] is True
assert report['batchid']
from shopdb.core.models.asset import Asset
from shopdb.core.models import BusinessUnit
asset = db.session.get(Asset, assetid)
assert asset.name == 'Renamed Machine'
assert asset.locationid == locationid
created = BusinessUnit.query.filter_by(businessunit='Large Parts').one()
assert asset.businessunitid == created.businessunitid
def test_commit_uses_existing_business_unit_when_present(client, db,
auth_headers):
from shopdb.core.models import BusinessUnit
existing = BusinessUnit(businessunit='Large Parts')
db.session.add(existing)
db.session.commit()
existing_id = existing.businessunitid
assetid, _ = _seed_machine(db)
# Already resolved, so no needs-create and no acknowledgment needed.
response = _commit(client, auth_headers)
assert response.status_code == 200, response.get_json()
from shopdb.core.models.asset import Asset
asset = db.session.get(Asset, assetid)
assert asset.businessunitid == existing_id
# No duplicate business unit was created.
assert BusinessUnit.query.filter_by(businessunit='Large Parts').count() == 1
def test_undo_reverses_a_committed_batch(client, db, auth_headers):
assetid, _ = _seed_machine(db)
committed = _commit(client, auth_headers, extra={'acknowledge': 'businessunit'})
batchid = committed.get_json()['data']['batchid']
from shopdb.core.models.asset import Asset
from shopdb.core.models import BusinessUnit
assert db.session.get(Asset, assetid).name == 'Renamed Machine'
response = client.post(UNDO, json={'batchid': batchid}, headers=auth_headers)
assert response.status_code == 200, response.get_json()
result = response.get_json()['data']
assert result['batchid'] == batchid
assert result['reverted'] > 0
assert result['conflicts'] == []
db.session.expire_all()
asset = db.session.get(Asset, assetid)
assert asset.name == 'Old Name'
assert asset.locationid is None
assert asset.businessunitid is None
# The importer-created business unit was hard-deleted on undo.
assert BusinessUnit.query.filter_by(businessunit='Large Parts').count() == 0
def test_undo_requires_batchid(client, db, auth_headers):
response = client.post(UNDO, json={}, headers=auth_headers)
assert response.status_code == 400
assert 'batchid' in response.get_json()['data']['error']['message'].lower()
def test_batches_lists_committed_then_undone(client, db, auth_headers):
_seed_machine(db)
committed = _commit(client, auth_headers, extra={'acknowledge': 'businessunit'})
batchid = committed.get_json()['data']['batchid']
listed = client.get(BATCHES, headers=auth_headers).get_json()['data']
batch = next(b for b in listed['batches'] if b['batchid'] == batchid)
assert batch['status'] == 'applied'
assert batch['kind'] == 'winc-machine-adoption'
client.post(UNDO, json={'batchid': batchid}, headers=auth_headers)
listed = client.get(BATCHES, headers=auth_headers).get_json()['data']
batch = next(b for b in listed['batches'] if b['batchid'] == batchid)
assert batch['status'] == 'undone'
assert batch['undoneat'] is not None
def test_winc_routes_require_admin(client, db, member_headers):
_seed_machine(db)
for url in (PREVIEW, COMMIT):
response = _upload(client, member_headers, url)
assert response.status_code in (401, 403)
assert client.get(BATCHES, headers=member_headers).status_code in (401, 403)
# --- an approved plan has to be the plan that gets applied --------------------
#
# The writer's drift check compared a packet against the plan it was built from,
# so it could never fail. "Approved" meant only that the admin ticked the right
# needs-create group NAMES - the plan itself could be anything.
def test_commit_refuses_a_plan_that_was_not_the_one_previewed(client, db,
auth_headers):
_seed_machine(db)
response = _commit(client, auth_headers,
extra={'acknowledge': 'businessunit'},
planhash='a' * 64)
assert response.status_code == 400
assert 'previewed' in response.get_json()['data']['error']['message']
def test_commit_refuses_without_a_planhash_at_all(client, db, auth_headers):
"""Committing without previewing is what made the approval vacuous."""
_seed_machine(db)
response = _upload(client, auth_headers, COMMIT,
extra={'acknowledge': 'businessunit'})
assert response.status_code == 400
assert 'planhash' in response.get_json()['data']['error']['message']
def test_a_preview_of_one_file_does_not_approve_another(client, db,
auth_headers):
"""The concrete attack: preview file A, commit file B."""
from shopdb.core.models import Asset
_seed_machine(db)
other_csv = (
'assetnum,description,location,eq1\r\n'
'W0083A,Something Else Entirely,WM-J3,lg parts\r\n'
)
approved = _upload(client, auth_headers, PREVIEW).get_json()['data']['planhash']
response = _upload(client, auth_headers, COMMIT, csv_text=other_csv,
extra={'acknowledge': 'businessunit',
'planhash': approved})
assert response.status_code == 400
assert Asset.query.filter_by(name='Something Else Entirely').first() is None