ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import the wilm agent store, WINC adoption work and batches CLI, with four fixes

Browse Source
This commit is contained in:
Cameron Proudlock
2026-09-03 08:16:38 -04:00
parent 4b21aa3887
commit b0bd73f62d
51 changed files with 5107 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
scripts/Start-ShopDBDev.ps1
View File

@@ -85,27 +85,27 @@ function Clear-InterruptedRun {
}
}
# A directory that exists but has no valid .git is a broken/partial clone -
# USUALLY. It is also what -TargetDir pointed at somebody's Documents folder
# looks like, and this ran Remove-Item -Recurse -Force on it without asking.
# For a script distributed as "save it anywhere and run it", a mistyped
# -TargetDir must not be able to delete a user's files.
#
# Only reclaim a directory this script can show it owns: empty, or carrying
# the marker written after a successful clone. Anything else stops.
if ((Test-Path $Target) -and -not (Test-Path (Join-Path $Target '.git'))) {
$ours = Test-Path (Join-Path $Target $script:CloneMarker)
$empty = -not (Get-ChildItem -LiteralPath $Target -Force |
Select-Object -First 1)
if ($ours -or $empty) {
Write-Host "==> Removing incomplete clone at $Target" -ForegroundColor Cyan
Remove-Item -Path $Target -Recurse -Force
}
else {
throw ("$Target already exists, is not empty, and was not created " +
"by this launcher. Refusing to delete it. Pass -TargetDir " +
'pointing somewhere else, or remove it yourself if you are sure.')
}
# A directory that exists but has no valid .git is a broken/partial clone -
# USUALLY. It is also what -TargetDir pointed at somebody's Documents folder
# looks like, and this ran Remove-Item -Recurse -Force on it without asking.
# For a script distributed as "save it anywhere and run it", a mistyped
# -TargetDir must not be able to delete a user's files.
#
# Only reclaim a directory this script can show it owns: empty, or carrying
# the marker written after a successful clone. Anything else stops.
if ((Test-Path $Target) -and -not (Test-Path (Join-Path $Target '.git'))) {
$ours = Test-Path (Join-Path $Target $script:CloneMarker)
$empty = -not (Get-ChildItem -LiteralPath $Target -Force |
Select-Object -First 1)
if ($ours -or $empty) {
Write-Host "==> Removing incomplete clone at $Target" -ForegroundColor Cyan
Remove-Item -Path $Target -Recurse -Force
}
else {
throw ("$Target already exists, is not empty, and was not created " +
"by this launcher. Refusing to delete it. Pass -TargetDir " +
'pointing somewhere else, or remove it yourself if you are sure.')
}
}
# A clone created by an elevated (Administrator) run is owned by

3
scripts/build-channel-installer.ps1
View File

@@ -6,7 +6,7 @@
Produces ONE self-contained installer package for the requested channel:
stable -> built from 'main', no DEV badge (SHOPDB_ENV_LABEL empty).
edge -> built from 'main' (the trunk), red DEV badge (SHOPDB_ENV_LABEL=DEV).
edge -> built from 'main' (the trunk), red DEV badge (SHOPDB_ENV_LABEL=DEV).
This is the distributable counterpart to scripts/dev-bootstrap.ps1 (which
stands a dev box up FROM the repo). Here the operator has no repo: the script
@@ -102,6 +102,7 @@ function Resolve-StableRef {
Write-Warning 'No v* tag in this repo; building stable from main.'
return 'main'
}
}
function Assert-LastExit([string]$What) {
if ($LASTEXITCODE -ne 0) { throw "$What failed (exit $LASTEXITCODE)" }

4
scripts/check-naming-and-style.sh
View File

@@ -35,6 +35,10 @@ EXCLUDES=(
--exclude-dir=.git
--exclude-dir=versions
--exclude-dir=staticdocs
# Local, gitignored scratch notes (see .gitignore devnotes/). Not product
# source, never committed, and free to hold pasted logs full of em-dashes
# and arrows - grep does not consult .gitignore, so exclude it explicitly.
--exclude-dir=devnotes
)
INCLUDES_CODE=(

30
scripts/dev-bootstrap.ps1
View File

@@ -12,7 +12,7 @@
passed, because the app itself runs entirely in the container.
2. Picks a channel:
stable -> pulls the 'stable' image (built from main), no DEV badge.
edge -> pulls the 'edge' image (built from main), red DEV badge.
edge -> pulls the 'edge' image (built from main), red DEV badge.
The two channels use different compose project names, host ports, and
named volumes, so they run side by side without colliding.
3. Generates a .env for the channel (dev-grade secrets, NOT for production)
@@ -282,20 +282,20 @@ try {
} else {
# Dev-grade secrets: generated locally, never for production. A real
# deploy fills .env from the wiki runbook, not from this script.
#
# Get-Random is System.Random: a 32-bit seed, and all four values came
# off one stream, so recovering the seed from any one of them recovers
# the other three. These sign JWTs for a stack a pilot may point at a
# real imported dataset, so they come from the CSPRNG instead. "Dev
# grade" is about rotation and provenance, not about being guessable.
$secret = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$jwt = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$dbPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
$rootPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
#
# Get-Random is System.Random: a 32-bit seed, and all four values came
# off one stream, so recovering the seed from any one of them recovers
# the other three. These sign JWTs for a stack a pilot may point at a
# real imported dataset, so they come from the CSPRNG instead. "Dev
# grade" is about rotation and provenance, not about being guessable.
$secret = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$jwt = [Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(48))
$dbPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
$rootPass = ([Convert]::ToBase64String(
[System.Security.Cryptography.RandomNumberGenerator]::GetBytes(18))) -replace '[^A-Za-z0-9]', 'x'
$apiPort = $config.ApiPort
$mysqlPort = $config.MysqlPort
$image = "$imageRepo`:$($config.ImageTag)"

149
scripts/site_imports/approval.py Normal file
View File

@@ -0,0 +1,149 @@
"""Reusable approval / promotion gate for site-import writers.
IMPORT-AGNOSTIC. This turns a dry-run plan into an "approval packet": a frozen
summary of what an import WOULD do, plus a content hash of the exact plan it
describes. A writer refuses to run unless it is handed a packet whose hash still
matches the plan in front of it. That is the whole mechanism: a human approves a
specific, frozen plan, and the writer can prove nothing drifted since.
The same packet shape serves every importer, and it is deliberately the seam the
undo work will reuse: a writer records the packet's batch id alongside the
journal it writes, so approval, write, and undo all reference one identity.
A packet is a plain dict (JSON-serialisable) so it can be printed, saved next to
an export, diffed between runs, and shown in a UI without any of this code:
{
'kind': caller-supplied label, e.g. 'winc-machine-adoption'
'planhash': sha256 of the canonicalised plan rows
'rowcount': number of rows in the plan
'counts': caller-supplied summary dict (what the report shows)
'needscreate': { group -> { name -> count } } # requires acknowledgment
'unresolved': { group -> { value -> count } } # blocks approval
'blocking': True if any unresolved entries exist
}
The writer's contract, in order:
1. rebuild the plan from the same inputs,
2. build a packet from it,
3. compare planhash to the approved packet - abort on mismatch,
4. refuse if packet['blocking'],
5. require explicit acknowledgment for every needscreate group,
6. only then write, stamping packet['batchid'] onto the journal.
"""
import hashlib
import json
import uuid
from collections import OrderedDict
from scripts.site_imports.resolver import tally, NEEDS_CREATE, UNRESOLVED
def plan_hash(plan):
"""Stable sha256 over the plan's rows.
Canonicalised by serialising each row as sorted-key JSON so the hash depends
only on content, not on dict ordering or whitespace. Two runs that produce
the same decisions produce the same hash; any changed decision changes it,
which is exactly what the writer checks before it touches the database.
"""
hasher = hashlib.sha256()
for row in plan:
canonical = json.dumps(row, sort_keys=True, default=str,
separators=(',', ':'))
hasher.update(canonical.encode('utf-8'))
hasher.update(b'\n')
return hasher.hexdigest()
def build_packet(kind, plan, counts, groups):
"""Build an approval packet from a plan.
kind a short label identifying the importer, stored on the packet.
plan the list of per-row dicts the writer will act on.
counts the caller's summary dict (shown to the human as-is).
groups how to read each field's status/value columns out of a row:
[ (group_name, status_column, value_column), ... ]
e.g. ('businessunit', 'businessunit_status', 'raw_eq1'). The value
column is what a human would act on - the raw export value for
UNRESOLVED (so they know what to alias) and the canonical name for
NEEDS_CREATE (so they know what would be created).
needscreate is keyed on the canonical name column when present; callers pass
the *_name column for that via the fourth tuple element when it differs from
the value column.
"""
needscreate = OrderedDict()
unresolved = OrderedDict()
for group in groups:
group_name, status_column, value_column = group[0], group[1], group[2]
name_column = group[3] if len(group) > 3 else value_column
unresolved_hits = tally(plan, status_column, value_column, UNRESOLVED)
needs_hits = tally(plan, status_column, name_column, NEEDS_CREATE)
if unresolved_hits:
unresolved[group_name] = unresolved_hits
if needs_hits:
needscreate[group_name] = needs_hits
return OrderedDict([
('kind', kind),
('batchid', str(uuid.uuid4())),
('planhash', plan_hash(plan)),
('rowcount', len(plan)),
('counts', OrderedDict(counts)),
('needscreate', needscreate),
('unresolved', unresolved),
('blocking', bool(unresolved)),
])
def packet_matches_plan(packet, plan):
"""True if the approved packet still describes this exact plan."""
return packet.get('planhash') == plan_hash(plan)
def format_packet(packet):
"""Human-readable rendering of a packet for the console or a UI panel."""
lines = []
lines.append('APPROVAL PACKET: %s' % packet.get('kind', '?'))
lines.append(' batch id : %s' % packet.get('batchid', '?'))
lines.append(' plan hash: %s' % packet.get('planhash', '?'))
lines.append(' rows : %d' % packet.get('rowcount', 0))
needscreate = packet.get('needscreate') or {}
if needscreate:
for group, items in needscreate.items():
total = sum(items.values())
lines.append('')
lines.append(' WARNING: this import would CREATE %d %s(s). '
'Confirm these are correct; you can edit them later '
'in Settings:' % (len(items), group))
for name, count in items.items():
lines.append(' + %-28s (%d rows)' % (name, count))
lines.append(' total new %s rows: %d' % (group, total))
unresolved = packet.get('unresolved') or {}
if unresolved:
for group, items in unresolved.items():
lines.append('')
lines.append(' BLOCKING: %d unresolved %s value(s) - add an alias '
'for each before importing:' % (len(items), group))
for value, count in items.items():
lines.append(' ? %-28s (%d rows)' % (value, count))
lines.append('')
lines.append(' status : %s' % (
'BLOCKED (resolve the values above)' if packet.get('blocking')
else 'READY for acknowledgment'))
return '\n'.join(lines)
def save_packet(packet, path):
with open(path, 'w', encoding='utf-8') as handle:
json.dump(packet, handle, indent=2)
def load_packet(path):
with open(path, encoding='utf-8') as handle:
return json.load(handle)

51
scripts/site_imports/clone_import_db.ps1 Normal file
View File

@@ -0,0 +1,51 @@
# Clone the live shopdb_flask database into a throwaway copy for import testing.
#
# Runs entirely inside the db container so the root password is read from the
# container's own environment and never typed on a command line or logged. The
# clone is a full structural + data copy, so a dry run or a writer pointed at it
# behaves exactly as it would against production - but nothing it does can touch
# the real data.
#
# pwsh scripts/site_imports/clone_import_db.ps1
# pwsh scripts/site_imports/clone_import_db.ps1 -Target shopdb_flask_import
#
# Point a tool at the clone by overriding DATABASE_URL for that one run, e.g.
# docker compose exec -e DATABASE_URL="mysql+pymysql://root:PW@db:3306/shopdb_flask_import?charset=utf8mb4" api ...
# (the winc dry run also honours DATABASE_URL, so --export works unchanged).
param(
[string]$Source = 'shopdb_flask',
[string]$Target = 'shopdb_flask_import'
)
$ErrorActionPreference = 'Stop'
Write-Host "Cloning '$Source' -> '$Target' inside the db container..."
# All work happens in the container. $MYSQL_ROOT_PASSWORD is already set there.
# MYSQL_PWD is honoured by both mysql and mysqldump, which keeps the password
# off the argument list (no -p on the command line to leak into ps or logs).
# Backticks around identifiers are MySQL's; placeholders are filled in by
# PowerShell below so the shell here sees plain literal database names.
$remote = @'
set -e
export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"
mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS \`__TARGET__\`; CREATE DATABASE \`__TARGET__\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
mysqldump -h 127.0.0.1 -u root --single-transaction --routines --triggers __SOURCE__ | mysql -h 127.0.0.1 -u root __TARGET__
mysql -h 127.0.0.1 -u root -N -e "SELECT CONCAT('assets in clone: ', COUNT(*)) FROM __TARGET__.assets;"
'@
$remote = $remote.Replace('__TARGET__', $Target).Replace('__SOURCE__', $Source)
# Strip CR so the container's sh sees clean LF-only lines (this file is CRLF on
# a Windows checkout, and sh treats a trailing \r as part of the argument).
$remote = $remote.Replace("`r", '')
# The db service name in docker-compose.yml is 'db'.
docker compose exec -T db sh -c $remote
if ($LASTEXITCODE -ne 0) {
throw "Clone failed (exit $LASTEXITCODE)."
}
Write-Host "Clone complete: $Target"

267
scripts/site_imports/journal.py Normal file
View File

@@ -0,0 +1,267 @@
"""Shared, import-agnostic journal + undo for reversible bulk imports.
IMPORT-AGNOSTIC. Nothing here knows about WINC, Maximo, machines, or any one
source. An import writer records the primitive mutations it makes - "set column
X on row Y of table T from old to new" (update) or "insert row Y into table T"
(insert) - each tied to a batch id, and a generic undo replays them in reverse.
That is the whole mechanism: importers emit primitives; undo reverses primitives.
Identity: one value threads approval, write, and undo. The approval packet's
`batchid` becomes the importbatches primary key, becomes the journal foreign key,
becomes the argument the "Undo import" button passes. Nothing to reconcile.
Contract (extends the approval packet's 6-step writer contract):
open_batch() -> after the packet passes, write the importbatches row.
record_update()/record_insert() -> append a journal entry in the SAME
transaction as the mutation it describes (ledger rule: the
entry and the change it records commit together).
undo_batch() -> reverse-replay this batch's entries, conservatively.
Undo is deliberately conservative and never destructive beyond what the import
owns:
- update reverts a cell ONLY if it still holds the value we wrote; a value a
human changed since is left as-is and reported as a conflict (never clobbered).
- insert hard-deletes ONLY the row this batch created, and only when nothing
outside the batch still references it; otherwise it is left and reported.
All access goes through a SQLAlchemy connection executing parameterised text, so
this reuses the same connection an importer already opened and pulls in no ORM
models - keeping it a true platform primitive any importer can share.
"""
from collections import namedtuple
from datetime import datetime, timezone
BATCH_APPLIED = 'applied'
BATCH_UNDONE = 'undone'
OP_UPDATE = 'update'
OP_INSERT = 'insert'
# Result of an undo pass, for a report and a UI.
UndoResult = namedtuple('UndoResult', ('batchid', 'reverted', 'conflicts'))
# A single thing undo could not safely reverse.
Conflict = namedtuple('Conflict', ('operation', 'table', 'targetid',
'column', 'reason'))
def _utcnow():
return datetime.now(timezone.utc).replace(tzinfo=None)
def _as_text(value):
"""Store every journalled value as text so a pre/post image compares the
same way it will read back, regardless of the column's declared type."""
return None if value is None else str(value)
def open_batch(connection, packet, createdby=None):
"""Write the importbatches row for an approved packet.
Call once, after the packet has been verified against the current plan and
acknowledged. Returns the batchid so the caller stamps it on every journal
entry it then records. Import-agnostic: the packet's own fields carry all
the identity this needs.
"""
from sqlalchemy import text
connection.execute(text(
'INSERT INTO importbatches '
'(batchid, kind, planhash, rowcount, createdat, createdby, status) '
'VALUES (:batchid, :kind, :planhash, :rowcount, :createdat, '
':createdby, :status)'),
{
'batchid': packet['batchid'],
'kind': packet.get('kind'),
'planhash': packet.get('planhash'),
'rowcount': packet.get('rowcount', 0),
'createdat': _utcnow(),
'createdby': createdby,
'status': BATCH_APPLIED,
})
return packet['batchid']
def record_update(connection, batchid, sequence, table, targetid, column,
oldvalue, newvalue):
"""Append an 'update' journal entry describing one cell change.
Record the pre-image (oldvalue) you read BEFORE applying the change and the
post-image (newvalue) you are about to write, in the same transaction as the
UPDATE itself.
"""
_insert_entry(connection, batchid, sequence, OP_UPDATE, table, targetid,
column, oldvalue, newvalue)
def record_insert(connection, batchid, sequence, table, targetid):
"""Append an 'insert' journal entry for a row this batch created.
Only ever emitted for a needs-create row the writer itself inserted, so undo
can hard-delete strictly what the import created.
"""
_insert_entry(connection, batchid, sequence, OP_INSERT, table, targetid,
None, None, None)
def _insert_entry(connection, batchid, sequence, operation, table, targetid,
column, oldvalue, newvalue):
from sqlalchemy import text
connection.execute(text(
'INSERT INTO importjournalentries '
'(batchid, sequence, operation, targettable, targetid, columnname, '
'oldvalue, newvalue, isreverted) '
'VALUES (:batchid, :sequence, :operation, :targettable, :targetid, '
':columnname, :oldvalue, :newvalue, 0)'),
{
'batchid': batchid,
'sequence': sequence,
'operation': operation,
'targettable': table,
'targetid': targetid,
'columnname': column,
'oldvalue': _as_text(oldvalue),
'newvalue': _as_text(newvalue),
})
def undo_batch(connection, batchid, references=None):
"""Reverse-replay an applied batch's journal. Returns an UndoResult.
references (optional): { table -> [(reftable, refcolumn), ...] } describing
which columns in other tables point at a created row's id. Used to decide
whether an inserted row is safe to hard-delete: if any reference OUTSIDE
this batch's own updates still points at it, the row is left and reported as
a conflict rather than deleted. Omit it to require that the batch's own
update-reverts have already cleared every reference (the common case, since a
created businessunit is only ever pointed at by the same batch's asset
updates).
Idempotent: a batch already 'undone' is a no-op. Entries are walked in
reverse sequence so inserts are removed only after the updates that pointed
at them are reverted.
"""
from sqlalchemy import text
batch = connection.execute(text(
'SELECT status FROM importbatches WHERE batchid = :batchid'),
{'batchid': batchid}).fetchone()
if batch is None:
raise ValueError('unknown batchid: %s' % batchid)
if batch[0] == BATCH_UNDONE:
return UndoResult(batchid, 0, [])
entries = connection.execute(text(
'SELECT entryid, sequence, operation, targettable, targetid, '
'columnname, oldvalue, newvalue FROM importjournalentries '
'WHERE batchid = :batchid AND isreverted = 0 '
'ORDER BY sequence DESC, entryid DESC'),
{'batchid': batchid}).fetchall()
reverted = 0
conflicts = []
for row in entries:
(entryid, _sequence, operation, table, targetid, column,
oldvalue, newvalue) = row
if operation == OP_UPDATE:
conflict = _revert_update(connection, table, targetid, column,
oldvalue, newvalue)
elif operation == OP_INSERT:
conflict = _revert_insert(connection, batchid, table, targetid,
references)
else:
conflict = Conflict(operation, table, targetid, column,
'unknown operation')
if conflict is None:
connection.execute(text(
'UPDATE importjournalentries SET isreverted = 1 '
'WHERE entryid = :entryid'), {'entryid': entryid})
reverted += 1
else:
conflicts.append(conflict)
connection.execute(text(
'UPDATE importbatches SET status = :status, undoneat = :undoneat '
'WHERE batchid = :batchid'),
{'status': BATCH_UNDONE, 'undoneat': _utcnow(), 'batchid': batchid})
return UndoResult(batchid, reverted, conflicts)
def _revert_update(connection, table, targetid, column, oldvalue, newvalue):
"""Revert one cell to its pre-image, but only if it still holds what we
wrote. Returns a Conflict if a human changed it since, else None."""
from sqlalchemy import text
pk = _primary_key(connection, table)
current = connection.execute(text(
'SELECT %s FROM %s WHERE %s = :targetid' % (column, table, pk)),
{'targetid': targetid}).fetchone()
if current is None:
return Conflict(OP_UPDATE, table, targetid, column, 'row missing')
if _as_text(current[0]) != _as_text(newvalue):
return Conflict(OP_UPDATE, table, targetid, column,
'value changed since import')
connection.execute(text(
'UPDATE %s SET %s = :oldvalue WHERE %s = :targetid'
% (table, column, pk)),
{'oldvalue': oldvalue, 'targetid': targetid})
return None
def _revert_insert(connection, batchid, table, targetid, references):
"""Hard-delete a row this batch created, only if nothing outside the batch
still references it. Returns a Conflict when it is not safe, else None."""
from sqlalchemy import text
pk = _primary_key(connection, table)
for reftable, refcolumn in (references or {}).get(table, []):
referencing = connection.execute(text(
'SELECT COUNT(*) FROM %s WHERE %s = :targetid'
% (reftable, refcolumn)), {'targetid': targetid}).fetchone()[0]
if referencing:
return Conflict(OP_INSERT, table, targetid, None,
'still referenced by %s.%s' % (reftable, refcolumn))
connection.execute(text(
'DELETE FROM %s WHERE %s = :targetid' % (table, pk)),
{'targetid': targetid})
return None
def _primary_key(connection, table):
"""Single-column primary key name for a table, via the live schema."""
from sqlalchemy import inspect
pk = inspect(connection).get_pk_constraint(table)
columns = pk.get('constrained_columns') or []
if len(columns) != 1:
raise ValueError(
'journal undo needs a single-column primary key on %s' % table)
return columns[0]
def format_undo_result(result):
"""Human-readable rendering of an UndoResult for the console or a UI panel."""
lines = []
lines.append('UNDO batch %s' % result.batchid)
lines.append(' reverted : %d entries' % result.reverted)
if result.conflicts:
lines.append(' conflicts: %d (left as-is, not reverted)'
% len(result.conflicts))
for conflict in result.conflicts:
target = conflict.table
if conflict.column:
target = '%s.%s' % (conflict.table, conflict.column)
lines.append(' ! %-8s %s #%s - %s' % (
conflict.operation, target, conflict.targetid,
conflict.reason))
else:
lines.append(' conflicts: none')
return '\n'.join(lines)

134
scripts/site_imports/resolver.py Normal file
View File

@@ -0,0 +1,134 @@
"""Shared four-state resolver vocabulary for site-import dry runs.
IMPORT-AGNOSTIC. Nothing here knows about machines, eq1, locations, or any one
site. Every site importer maps a raw exported value onto a database row, and
every one of those mappings lands in exactly one of four states:
RESOLVED the value maps to a row that already exists; safe to set.
BLANK there is nothing to set (empty cell, or a deliberately
unmappable value); leave the target field as-is.
NEEDS_CREATE the value maps to a valid canonical name, but that row does
not exist yet; a writer may create it, WITH acknowledgment.
UNRESOLVED the value has no mapping at all; a human must add one before
any writer runs. This is the state that blocks approval.
Keeping this vocabulary in one place is what lets a promotion/approval helper,
an undo journal, and any number of importers all speak the same language. Add
new importers by writing their lookups and alias maps; reuse these primitives to
turn a raw value into a Resolution.
"""
from collections import OrderedDict, namedtuple
# The four terminal states a single field resolution can end in.
RESOLVED = 'resolved'
BLANK = 'blank'
NEEDS_CREATE = 'needs-create'
UNRESOLVED = 'unresolved'
# status: one of the four above.
# id: the database id when RESOLVED, else None.
# name: the canonical target name when known (RESOLVED / NEEDS_CREATE), else
# None. Carried so callers can report and, later, create by name.
Resolution = namedtuple('Resolution', ('status', 'id', 'name'))
def norm(value):
"""Lowercase, stripped form used for every name comparison and alias key.
Centralised so a lookup table, an alias map, and the value being resolved
are always normalised the same way. If this ever changes, it changes for
everyone at once, which is the point.
"""
return (value or '').strip().lower()
def load_lookup(connection, table, id_column, name_column, display=None):
"""norm(name) -> id for every row of any table. One loader, every importer.
This is the "match against what is actually in the database" primitive. It
reads the live table at call time, so a value a human adds in Settings is
visible on the next run with no code change: a name that exists resolves,
one that does not is NEEDS_CREATE or UNRESOLVED. Every site importer maps its
exported values onto some reference table (locations, businessunits, vendors,
...) and they should all load that table the same way rather than each
hand-rolling a SELECT.
Pass a dict as `display` to also collect norm(name) -> original-cased name,
so a report can echo the name the way the database spells it rather than the
normalised key.
First writer wins on a duplicate normalised name, which is deterministic and
matches how the importers resolve: the ambiguity is a data problem to report,
not something the loader should silently reorder.
"""
from sqlalchemy import text
out = {}
rows = connection.execute(text(
'SELECT %s, %s FROM %s' % (id_column, name_column, table)))
for ident, name in rows:
key = norm(name)
if key:
out.setdefault(key, ident)
if display is not None:
display.setdefault(key, (name or '').strip())
return out
def resolve_name(name, lookup_by_name, missing=UNRESOLVED):
"""Resolve a value by direct name match against a lookup table.
lookup_by_name maps norm(name) -> database id. A blank name is BLANK. A hit
is RESOLVED with the id. A miss is `missing` - callers pass UNRESOLVED when a
human must supply a mapping, or NEEDS_CREATE when the name is trusted and a
writer may create it.
"""
if not (name or '').strip():
return Resolution(BLANK, None, None)
ident = lookup_by_name.get(norm(name))
if ident is not None:
return Resolution(RESOLVED, ident, name)
return Resolution(missing, None, name)
def resolve_alias(raw, alias_map, lookup_by_name, missing=UNRESOLVED):
"""Resolve a dirty raw value through an alias map, then a lookup table.
The alias-first path: a raw export value (possibly truncated or oddly cased)
is normalised and looked up in alias_map to get a canonical name, which is
then resolved against lookup_by_name.
raw is blank -> BLANK (nothing to set)
raw not in alias_map -> missing (default UNRESOLVED: add an alias)
alias maps to None -> BLANK (deliberately unmappable value)
canonical exists in lookup -> RESOLVED
canonical not yet in lookup -> NEEDS_CREATE
"""
if not (raw or '').strip():
return Resolution(BLANK, None, None)
key = norm(raw)
if key not in alias_map:
return Resolution(missing, None, None)
canonical = alias_map[key]
if canonical is None:
return Resolution(BLANK, None, None)
return resolve_name(canonical, lookup_by_name, missing=NEEDS_CREATE)
def tally(plan, status_column, value_column, status):
"""Count rows in a plan whose status_column equals `status`, by value.
Returns an OrderedDict of value -> count, highest count first. Shared by the
dry-run report and the approval packet so the numbers a human reads and the
numbers a writer enforces come from one place.
"""
counts = {}
for row in plan:
if row.get(status_column) == status:
value = row.get(value_column) or ''
counts[value] = counts.get(value, 0) + 1
ordered = OrderedDict()
for value, count in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
ordered[value] = count
return ordered

159
scripts/site_imports/suggest.py Normal file
View File

@@ -0,0 +1,159 @@
"""Conservative, explainable match suggestions for UNRESOLVED values.
IMPORT-AGNOSTIC and REPORT-ONLY. This never resolves anything and never writes.
Given a raw value that the normal rules could not map, it proposes zero or more
candidate names that already exist in a lookup table, each with a reason and a
confidence, so a human can accept one into that importer's alias map. Nothing
here is ever applied automatically - a silent wrong match is the worst failure
mode in dirty asset data, so every suggestion has to be acknowledged.
Two tiers, tried in order, stopping at the first that yields anything:
1. Bounded deterministic transforms (confidence 'high'). These are rules that
fit the shapes actually seen in the data - a bay code with a slash is two
bays (T6/U6 -> T6, U6), a parenthetical is a note (LD9W(GE90) -> LD9W), a
decimal is a sub-bay (M6.3 -> M6), a trailing range is its first member
(L1-4 -> L1). Each transform is a named rule, not a fuzzer, so the reason
printed next to a suggestion tells a human exactly why it was proposed.
2. A single edit-distance-1 fallback (confidence 'low'). Only if no transform
matched, and only for a one-character difference against an existing name.
Bounded to distance 1 on purpose: it catches a typo or a dropped letter and
nothing looser, so it cannot quietly invent a plausible-looking wrong bay.
The caller supplies the same normalisation and the lookup it resolves against, so
suggestions are always checked against what is really in the database.
"""
import re
from collections import namedtuple
from scripts.site_imports.resolver import norm
# candidate: the existing name we think the raw value meant.
# reason: which rule proposed it (shown to the human).
# confidence: 'high' for a deterministic transform, 'low' for edit-distance.
Suggestion = namedtuple('Suggestion', ('candidate', 'reason', 'confidence'))
_PAREN = re.compile(r'\s*\([^)]*\)')
def _candidates_from_transforms(stripped):
"""Yield (transformed_name, reason) from the bounded deterministic rules.
Operates on the value AFTER the importer has removed its own prefix (e.g.
WM-). Each rule yields plain names; the caller checks each against the live
lookup, so a rule that produces a non-existent name simply contributes no
suggestion.
"""
# A parenthetical note: LD9W(GE90) -> LD9W
if '(' in stripped:
yield (_PAREN.sub('', stripped).strip(), 'dropped parenthetical note')
# A slash means two bays: T6/U6 -> T6 and U6. A bare second half (9/10)
# inherits the first half's letter prefix (T9/10 -> T9, T10).
if '/' in stripped:
left, right = (part.strip() for part in stripped.split('/', 1))
if left:
yield (left, 'first of a slash-joined pair')
if right:
yield (right, 'second of a slash-joined pair')
prefix = re.match(r'^([A-Za-z]+)', left)
if prefix and re.match(r'^\d', right):
yield (prefix.group(1) + right,
'second of a slash-joined pair, prefix carried')
# A decimal sub-bay: M6.3 -> M6
if '.' in stripped:
yield (stripped.split('.', 1)[0].strip(), 'dropped decimal sub-bay')
# A trailing numeric range: L1-4 -> L1
dash = re.match(r'^([A-Za-z]*\d+)-\d+$', stripped)
if dash:
yield (dash.group(1), 'first of a numeric range')
def _edit_distance_at_most_one(left, right):
"""True if `left` and `right` differ by at most one insert/delete/replace.
Bounded on purpose. A length gap over one is an immediate no; otherwise walk
both once and allow a single mismatch. Cheap and strict - it is a typo
catcher, not a similarity search.
"""
if left == right:
return True
length_left, length_right = len(left), len(right)
if abs(length_left - length_right) > 1:
return False
if length_left == length_right:
return sum(a != b for a, b in zip(left, right)) == 1
# Ensure left is the shorter one, then try to match with one insertion.
if length_left > length_right:
left, right = right, left
length_left, length_right = length_right, length_left
index_left = index_right = 0
skipped = False
while index_left < length_left and index_right < length_right:
if left[index_left] == right[index_right]:
index_left += 1
index_right += 1
elif skipped:
return False
else:
skipped = True
index_right += 1
return True
# A low-confidence edit-distance pass can match many short bay codes at once,
# which is noise, not help. Cap how many it emits so a human sees the few most
# plausible typo-fixes and nothing drowns the high-confidence transforms.
_MAX_LOW_CONFIDENCE = 3
def suggest(raw_value, stripped_value, lookup_by_name, display=None):
"""Suggestions for one unresolved value. Empty list means "no idea".
raw_value the original export string (for reporting only).
stripped_value the value after the importer removed its own prefix; this is
what the transforms and the edit-distance search operate on.
lookup_by_name norm(name) -> id, the live table to match against.
display optional norm(name) -> original-cased name, so a suggestion
echoes the database's spelling instead of the normalised key.
Deterministic transforms first; only if none of them hits an existing name
does the edit-distance-1 fallback run, and that fallback is capped. High-
confidence suggestions are never diluted by low-confidence ones.
"""
display = display or {}
def spell(key):
return display.get(key, key)
results = []
seen = set()
for candidate, reason in _candidates_from_transforms(stripped_value):
candidate = (candidate or '').strip()
if not candidate:
continue
key = norm(candidate)
if key in lookup_by_name and key not in seen:
seen.add(key)
results.append(Suggestion(spell(key), reason, 'high'))
if results:
return results
target = norm(stripped_value)
if not target:
return results
low = []
for existing_key in lookup_by_name:
if existing_key not in seen and _edit_distance_at_most_one(
target, existing_key):
seen.add(existing_key)
low.append(Suggestion(spell(existing_key),
'one-character difference', 'low'))
# Only surface the capped set; too many one-off matches is noise.
if len(low) <= _MAX_LOW_CONFIDENCE:
results.extend(sorted(low))
return results

67
scripts/site_imports/winc/README.md Normal file
View File

@@ -0,0 +1,67 @@
# WINC machine-adoption import (CLI)
Adopts machines from a Maximo asset export (validated against the WINC / West
Manufacturing site). It only updates machines that already exist; it never
creates canonical per-table rows. Every commit is journaled so it can be undone
as one batch.
Run all commands from the repo root. Each reads `.env` and connects to the
configured database.
## 1. Preview (read-only)
```
python -m scripts.site_imports.winc.dry_run --export EXPORT.csv
```
| switch | what it does |
| --- | --- |
| `--export PATH` | the Maximo ASSET export CSV to analyze (required) |
| `--rows` | print the per-row plan, not just the summary |
| `--csv PATH` | write the full plan to a CSV so runs can be diffed |
Nothing is written. Use this to see what would change and which values are
unresolved before committing.
## 2. Commit (writes, reversibly)
```
python -m scripts.site_imports.winc.writer --export EXPORT.csv --commit --acknowledge businessunit
```
| switch | what it does |
| --- | --- |
| `--export PATH` | the same export CSV (required) |
| `--commit` | actually write; without it this is a dry run |
| `--acknowledge GROUP` | approve a needs-create group (repeatable), e.g. `businessunit` |
| `--createdby NAME` | who approved the import; stamped on the batch |
On success it prints the new `batchid` and the exact undo command.
## 3. List batches
```
python -m scripts.site_imports.winc.batches
```
| switch | what it does |
| --- | --- |
| `--limit N` | show at most N batches, newest first (default 50) |
| `--json` | emit JSON instead of a table |
Read-only. Use it to find the `batchid` to undo.
## 4. Undo a batch
```
python -m scripts.site_imports.winc.undo --batch BATCHID --commit
```
| switch | what it does |
| --- | --- |
| `--batch BATCHID` | the batch to reverse (required) |
| `--commit` | actually reverse; without it this is a no-op |
Reversal is conservative: a cell a human changed since the import is left as-is
and reported, and a created business unit is deleted only when nothing outside
the batch still references it.

0
scripts/site_imports/winc/__init__.py Normal file
View File

67
scripts/site_imports/winc/aliases.py Normal file
View File

@@ -0,0 +1,67 @@
"""Alias and normalization tables for the WINC machine adoption dry-run.
This file is DATA, not logic. It is the single place a human edits when the
analyzer reports an unresolved value. Keep it boring: plain dicts, one entry per
line, lowercase keys. The analyzer lowercases and strips before it looks a value
up here, so keys here must be lowercase and already stripped.
Two independent problems get solved here:
1. eq1 -> business unit. The May export's eq1 column is a free-text value-stream
label (LG PARTS, SM SPOOLS, Large Spoo, HPTCR ...). It is dirty: truncated at
ten characters, inconsistently cased, and it does NOT match anything in the
live businessunits table (that table is a list of offices auto-created by an
earlier printer import). So every eq1 value must be mapped by hand to the
canonical business-unit name we want on the asset. A canonical name that does
not yet exist in the database is reported by the analyzer as "would need to
be created" - the analyzer never invents one and never writes one.
2. location code overrides. Most WM-<bay> codes resolve by simply dropping the
WM- prefix and matching a location by name (WM-J3 -> J3). A handful do not,
because the bay code carries a slash, a note in parentheses, or a room name
the location table spells differently. Those exceptions - and only those -
live here, mapping the RAW export location string to the exact locationname
already in the database.
"""
# eq1 (lowercased, stripped) -> canonical business unit name.
# The value on the right is what we intend to store; the analyzer checks whether
# it already exists in businessunits and flags the ones that do not.
EQ1_TO_BUSINESSUNIT = {
'lg parts': 'Large Parts',
'lg part': 'Large Parts',
'large part': 'Large Parts',
'sm spools': 'Small Spools',
'sm spool': 'Small Spools',
'large spoo': 'Large Spools',
'lg spools': 'Large Spools',
'lg spool': 'Large Spools',
'hptcr': 'HPT Rotor',
'bliskcr': 'Blisk Rotor',
'blisk': 'Blisk Rotor',
'leap x': 'LEAP-X',
}
# Raw export location string (lowercased, stripped) -> exact locationname in DB.
# Only for codes that do NOT resolve by dropping the WM- prefix. Leave a value
# as None to say "known to be unmappable, do not touch location for this row".
#
# The entries below the two None rows were promoted from the dry-run's
# high-confidence suggestions and verified to exist in the locations table. A
# slash-joined bay code (T6/U6) names two adjacent bays, but an asset carries a
# single locationid, so each maps to the FIRST (primary) bay - the deterministic
# choice a human confirmed. LD7W and LD6W are intentionally absent: they had no
# existing match and need a location created before they can be aliased.
LOCATION_OVERRIDES = {
'wm': None,
'wm-toolroom': None,
'wm-t6/u6': 'T6',
'wm-t9/10': 'T9',
'wm-u9/10': 'U9',
'wm-t4/u4': 'T4',
'wm-ld9w(ge90)': 'LD9W',
'wm-m6.3': 'M6',
'wm-l1-4': 'L1',
'wm-ld7w': None,
'wm-ld6w': None,
}

91
scripts/site_imports/winc/batches.py Normal file
View File

@@ -0,0 +1,91 @@
"""WINC machine adoption - BATCHES (list applied/undone import batches).
A read-only companion to writer.py and undo.py: it prints the WINC batches
newest-first so an operator can find a batchid to feed `--batch` into the undo
CLI. Like the other WINC commands it is WINC-named only for discoverability; the
underlying importbatches table is import-agnostic, so this simply filters to the
winc-machine-adoption kind.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
from pathlib import Path
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from sqlalchemy import text
from scripts.site_imports.winc.dry_run import build_app_connection
_KIND = 'winc-machine-adoption'
def _iso(value):
# PyMySQL hands back datetime objects; SQLite hands back the stored ISO
# string. Normalise both to an ISO string so the output is stable.
if value is None:
return '-'
if hasattr(value, 'isoformat'):
return value.isoformat()
return str(value)
def list_batches(connection, limit):
rows = connection.execute(text(
'SELECT batchid, kind, rowcount, createdat, createdby, status, '
'undoneat FROM importbatches WHERE kind = :kind '
'ORDER BY createdat DESC LIMIT :limit'),
{'kind': _KIND, 'limit': limit}).fetchall()
return [
{'batchid': r[0], 'kind': r[1], 'rowcount': r[2],
'createdat': _iso(r[3]), 'createdby': r[4], 'status': r[5],
'undoneat': _iso(r[6])}
for r in rows
]
def _print_table(batches):
if not batches:
print('No WINC import batches found.')
return
header = '%-36s %-8s %5s %-20s %-16s %s' % (
'batchid', 'status', 'rows', 'createdat', 'createdby', 'undoneat')
print(header)
print('-' * len(header))
for batch in batches:
print('%-36s %-8s %5s %-20s %-16s %s' % (
batch['batchid'], batch['status'], batch['rowcount'],
batch['createdat'], batch['createdby'] or '-', batch['undoneat']))
def main(argv=None):
parser = argparse.ArgumentParser(
description='List applied/undone WINC import batches, newest first.')
parser.add_argument('--limit', type=int, default=50,
help='Maximum number of batches to show (default 50).')
parser.add_argument('--json', action='store_true',
help='Emit the batches as JSON instead of a table.')
args = parser.parse_args(argv)
if args.limit < 1:
parser.error('--limit must be a positive integer')
_app, connection = build_app_connection()
batches = list_batches(connection, args.limit)
if args.json:
import json
print(json.dumps(batches, indent=2))
else:
_print_table(batches)
return 0
if __name__ == '__main__':
sys.exit(main())

385
scripts/site_imports/winc/dry_run.py Normal file
View File

@@ -0,0 +1,385 @@
"""Dry-run analyzer for the WINC (West Manufacturing) machine adoption.
READ ONLY. This script opens the database, reads the May asset export, decides
what an import WOULD change, and prints a report. It never writes. Nothing here
commits, updates, or inserts - that is a separate tool, written only once this
report reads clean and a human has signed off on it.
What it does, in order:
1. Load every existing machine's asset number and build the crosswalk key
(the workstation number, e.g. 0083A) the export will be matched on.
2. Read the export with a real CSV parser (quoted commas, BOM) and, for each
row that carries a workstation number, decide:
- does it match an existing machine? (in scope vs skipped)
- what name would we set? (export description)
- what location would we set? (WM-<bay> -> locations, via
aliases.LOCATION_OVERRIDES for
the exceptions)
- what business unit would we set? (eq1 -> aliases.EQ1_TO_
BUSINESSUNIT -> businessunits)
3. Classify each field resolution as RESOLVED, BLANK (nothing to set, leave
as-is), NEEDS-CREATE (target value is valid but not yet in the database),
or UNRESOLVED (no alias entry - a human must add one to aliases.py).
4. Print a summary and, with --rows, a per-row table. With --csv PATH it also
writes the full plan so the numbers can be diffed between runs.
Run it against the real database read-only:
python -m scripts.site_imports.winc.dry_run --export "exports/ASSET_current May26.csv"
python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --rows
python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --csv plan.csv
The design goal is reliability over cleverness: one match rule, one location
rule, one business-unit rule, every exception in aliases.py, and a report that
tells a human exactly what is unresolved so the alias table can be completed
before anyone runs the writer.
"""
import argparse
import csv
import os
import re
import sys
from collections import OrderedDict
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
# Same reasoning as scripts/export_catalog.py: a plain `python -m` run gets
# neither wsgi.py's nor the flask CLI's .env load, so DATABASE_URL would be
# unset and config would silently fall back to a hardcoded localhost URL. Load
# it here before anything reads config.
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from scripts.site_imports.approval import build_packet, format_packet
from scripts.site_imports.suggest import suggest
from scripts.site_imports.resolver import (
BLANK,
NEEDS_CREATE,
RESOLVED,
Resolution,
UNRESOLVED,
load_lookup,
norm as _norm,
resolve_alias,
resolve_name,
)
from scripts.site_imports.winc.aliases import (
EQ1_TO_BUSINESSUNIT,
LOCATION_OVERRIDES,
)
# WM- is the only bay prefix in the WINC export; a location code resolves by
# dropping it and matching a location by name. Everything else is an override.
_WM_PREFIX = re.compile(r'^WM-?', re.IGNORECASE)
# A workstation number is a LEADING run of digits with an optional single letter
# suffix. The export writes it clean (W0083A); the machines table stores the
# same number but often with a trailing descriptor (0083 Debur, 0083A Inspect).
# So the key is the leading token on both sides, matched unanchored - anchoring
# with $ would drop every existing machine whose name carries a descriptor and
# silently shrink the in-scope set. The leading W is the single biggest gotcha
# in this data, so it is stripped here in exactly one place.
_WSNUM = re.compile(r'^(\d{3,5}[A-Z]?)')
def workstation_key(assetnum):
"""The crosswalk key (e.g. '0083A') for an asset number, or None.
Strips a leading W and keeps the leading run of digits plus an optional
single trailing letter, ignoring any descriptor that follows. Returns None
for anything that is not a workstation number (blank rows, facilities codes,
named test machines) so non-machine rows fall out of scope on their own
rather than being force-fit.
"""
text = (assetnum or '').strip().upper()
if text.startswith('W'):
text = text[1:]
match = _WSNUM.match(text)
return match.group(1) if match else None
def load_existing_machine_keys(connection):
"""Map workstation key -> list of existing machine asset numbers.
A list, not a single value, because the machines table has a few numbers
that collide once reduced to their workstation key (0083 exists as
'0083 Debur' and '0083 Inspection'). The analyzer reports those as ambiguous
rather than guessing which one the export row means.
"""
from sqlalchemy import text
keys = OrderedDict()
rows = connection.execute(text(
"SELECT a.assetnumber "
"FROM assets a JOIN machines m ON m.assetid = a.assetid"))
for (assetnumber,) in rows:
key = workstation_key(assetnumber)
if key:
keys.setdefault(key, []).append(assetnumber)
return keys
def load_locations(connection, display=None):
"""locationname (normalized) -> locationid, from the live locations table."""
return load_lookup(connection, 'locations', 'locationid', 'locationname',
display=display)
def load_businessunits(connection):
"""businessunit (normalized) -> businessunitid, from the live table."""
return load_lookup(connection, 'businessunits', 'businessunitid',
'businessunit')
def resolve_location(raw_location, locations_by_name):
"""Resolution for a raw WM bay code.
Rule: an override in aliases wins; otherwise drop the WM- prefix and match a
location by name. A blank export cell means "leave the asset's location as
is" and is reported as BLANK, not UNRESOLVED, because there is nothing to
set and nothing for a human to fix. Overrides are checked first because they
carry the exceptions the plain prefix rule cannot handle.
"""
raw = (raw_location or '').strip()
if not raw:
return Resolution(BLANK, None, None)
if _norm(raw) in LOCATION_OVERRIDES:
return resolve_alias(raw, LOCATION_OVERRIDES, locations_by_name)
name = _WM_PREFIX.sub('', raw).strip()
return resolve_name(name, locations_by_name)
def resolve_businessunit(raw_eq1, businessunits_by_name):
"""Resolution for an eq1 value stream.
A blank eq1 is BLANK (leave the asset alone). A non-blank value must have an
alias entry; without one it is UNRESOLVED so a human adds it to aliases.py. A
mapped canonical name that does not yet exist in businessunits is
NEEDS-CREATE - flagged, never created here.
"""
return resolve_alias(raw_eq1, EQ1_TO_BUSINESSUNIT, businessunits_by_name)
def read_export(path):
"""Rows of the export as dicts. utf-8-sig handles the BOM; the csv module
handles the quoted commas embedded in some descriptions."""
with open(path, newline='', encoding='utf-8-sig') as handle:
return list(csv.DictReader(handle))
def parse_export_text(text):
"""Rows of an export given its raw CSV text (same shape as read_export).
Split out so an API route can hand the analyzer an uploaded file's bytes
without writing them to disk, while the CLI keeps reading from a path.
"""
import io
return list(csv.DictReader(io.StringIO(text)))
def analyze(connection, export_path):
"""Build the per-row plan from a file path. Returns (plan, summary, lookups)."""
return analyze_rows(connection, read_export(export_path))
def analyze_rows(connection, export_rows):
"""Build the per-row plan. Returns (plan_rows, summary_dict, lookups).
Takes already-parsed export rows so the same analysis serves both the CLI
(which reads a path) and the API route (which reads uploaded bytes).
"""
existing = load_existing_machine_keys(connection)
location_names = {}
locations = load_locations(connection, display=location_names)
businessunits = load_businessunits(connection)
plan = []
summary = OrderedDict([
('export_rows', len(export_rows)),
('non_workstation_rows', 0),
('in_scope', 0),
('skipped_no_existing_machine', 0),
('ambiguous_match', 0),
('location_' + RESOLVED, 0),
('location_' + BLANK, 0),
('location_' + NEEDS_CREATE, 0),
('location_' + UNRESOLVED, 0),
('businessunit_' + RESOLVED, 0),
('businessunit_' + BLANK, 0),
('businessunit_' + NEEDS_CREATE, 0),
('businessunit_' + UNRESOLVED, 0),
])
for row in export_rows:
assetnum = (row.get('assetnum') or '').strip()
key = workstation_key(assetnum)
if key is None:
summary['non_workstation_rows'] += 1
continue
matches = existing.get(key, [])
if not matches:
summary['skipped_no_existing_machine'] += 1
continue
summary['in_scope'] += 1
ambiguous = len(matches) > 1
if ambiguous:
summary['ambiguous_match'] += 1
loc = resolve_location(row.get('location'), locations)
bu = resolve_businessunit(row.get('eq1'), businessunits)
summary['location_' + loc.status] += 1
summary['businessunit_' + bu.status] += 1
plan.append(OrderedDict([
('assetnum', assetnum),
('workstation_key', key),
('matched_asset', '|'.join(matches)),
('ambiguous', 'yes' if ambiguous else ''),
('name', (row.get('description') or '').strip()),
('raw_location', (row.get('location') or '').strip()),
('location_status', loc.status),
('locationid', loc.id if loc.id is not None else ''),
('location_name', loc.name or ''),
('raw_eq1', (row.get('eq1') or '').strip()),
('businessunit_status', bu.status),
('businessunitid', bu.id if bu.id is not None else ''),
('businessunit_name', bu.name or ''),
]))
return plan, summary, {'locations': locations,
'location_names': location_names,
'businessunits': businessunits}
def print_summary(summary):
print('=' * 60)
print('WINC machine adoption - DRY RUN (no changes written)')
print('=' * 60)
for label, value in summary.items():
print(' %-32s %6d' % (label.replace('_', ' '), value))
def print_unresolved(plan, lookups=None):
loc = OrderedDict()
bu = OrderedDict()
for row in plan:
if row['location_status'] == UNRESOLVED:
loc[row['raw_location']] = loc.get(row['raw_location'], 0) + 1
if row['businessunit_status'] == UNRESOLVED:
bu[row['raw_eq1']] = bu.get(row['raw_eq1'], 0) + 1
needs_loc = OrderedDict()
needs_bu = OrderedDict()
for row in plan:
if row['location_status'] == NEEDS_CREATE:
needs_loc[row['location_name']] = needs_loc.get(row['location_name'], 0) + 1
if row['businessunit_status'] == NEEDS_CREATE:
needs_bu[row['businessunit_name']] = needs_bu.get(row['businessunit_name'], 0) + 1
if loc:
print('\nUNRESOLVED locations (add to aliases.LOCATION_OVERRIDES):')
location_lookup = (lookups or {}).get('locations') or {}
location_names = (lookups or {}).get('location_names') or {}
for value, count in sorted(loc.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
stripped = _WM_PREFIX.sub('', value).strip()
for hint in suggest(value, stripped, location_lookup,
display=location_names):
print(' -> suggest "%s" (%s, confidence %s)'
% (hint.candidate, hint.reason, hint.confidence))
if bu:
print('\nUNRESOLVED eq1 (add to aliases.EQ1_TO_BUSINESSUNIT):')
for value, count in sorted(bu.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
if needs_loc:
print('\nNEEDS-CREATE locations (mapped, but not yet in locations):')
for value, count in sorted(needs_loc.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
if needs_bu:
print('\nNEEDS-CREATE business units (mapped, but not yet in businessunits):')
for value, count in sorted(needs_bu.items(), key=lambda kv: -kv[1]):
print(' %5d %s' % (count, value))
def print_rows(plan):
print('\nPER-ROW PLAN:')
header = ('assetnum', 'matched_asset', 'name', 'raw_location',
'location_status', 'raw_eq1', 'businessunit_status')
print(' ' + ' | '.join(header))
for row in plan:
print(' ' + ' | '.join(str(row[h]) for h in header))
def write_csv(plan, path):
if not plan:
return
with open(path, 'w', newline='', encoding='utf-8') as handle:
writer = csv.DictWriter(handle, fieldnames=list(plan[0].keys()))
writer.writeheader()
writer.writerows(plan)
def build_app_connection():
"""Yield a live SQLAlchemy connection inside an app context.
Returns (app, connection). Caller is responsible for closing nothing - the
connection lives for the app context and the process exits right after.
"""
from shopdb import create_app
from shopdb.api import db
app = create_app()
context = app.app_context()
context.push()
return app, db.session.connection()
def main(argv=None):
parser = argparse.ArgumentParser(
description='Dry-run analyzer for WINC machine adoption. Read only.')
parser.add_argument('--export', required=True,
help='Path to the ASSET export CSV.')
parser.add_argument('--rows', action='store_true',
help='Print the full per-row plan.')
parser.add_argument('--csv', dest='csv_out',
help='Write the full per-row plan to this CSV path.')
parser.add_argument('--packet', action='store_true',
help='Print the approval packet (hash, needs-create '
'and blocking summary) a writer would enforce.')
args = parser.parse_args(argv)
if not os.path.exists(args.export):
parser.error('export file not found: %s' % args.export)
_app, connection = build_app_connection()
plan, summary, lookups = analyze(connection, args.export)
print_summary(summary)
print_unresolved(plan, lookups)
if args.rows:
print_rows(plan)
if args.csv_out:
write_csv(plan, args.csv_out)
print('\nWrote per-row plan: %s (%d rows)' % (args.csv_out, len(plan)))
if args.packet:
packet = build_packet(
'winc-machine-adoption', plan, summary,
[('businessunit', 'businessunit_status', 'raw_eq1',
'businessunit_name'),
('location', 'location_status', 'raw_location', 'location_name')])
print('\n' + format_packet(packet))
return 0
if __name__ == '__main__':
sys.exit(main())

55
scripts/site_imports/winc/undo.py Normal file
View File

@@ -0,0 +1,55 @@
"""WINC machine adoption - UNDO (reverse an applied import batch).
A thin CLI over the shared, import-agnostic journal.undo_batch. It is WINC-named
only for discoverability; the actual work is generic, so the same reversal serves
any importer that wrote through the journal. Reverting is conservative: a cell a
human changed since the import is left as-is and reported, and a created business
unit is deleted only when nothing outside the batch still references it.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
from pathlib import Path
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from scripts.site_imports import journal
from scripts.site_imports.winc.dry_run import build_app_connection
from scripts.site_imports.winc.writer import _UNDO_REFERENCES
def main(argv=None):
parser = argparse.ArgumentParser(
description='Undo an applied WINC import batch (reversible journal).')
parser.add_argument('--batch', required=True,
help='The batch id to reverse.')
parser.add_argument('--commit', action='store_true',
help='Actually reverse. Without it, this is a no-op.')
args = parser.parse_args(argv)
if not args.commit:
print('DRY: pass --commit to reverse batch %s' % args.batch)
return 0
_app, connection = build_app_connection()
from shopdb.api import db
try:
result = journal.undo_batch(connection, args.batch,
references=_UNDO_REFERENCES)
db.session.commit()
except Exception:
db.session.rollback()
raise
print(journal.format_undo_result(result))
return 0
if __name__ == '__main__':
sys.exit(main())

278
scripts/site_imports/winc/writer.py Normal file
View File

@@ -0,0 +1,278 @@
"""WINC machine adoption - WRITER (applies the approved plan, reversibly).
This is the write half of the WINC import; scripts/site_imports/winc/dry_run.py
is the read-only half. The writer never invents its own view of the data: it
rebuilds the exact same plan the dry-run produced, verifies that plan against the
approval packet the human signed off on, and only then writes - recording every
change to the shared import journal so the whole batch can be undone.
The contract, in order (the one documented in approval.py, made concrete here):
1. rebuild the plan from the same export + live DB,
2. build a packet from it,
3. compare planhash to the approved packet - abort on mismatch (drift),
4. refuse if the packet is blocking (any unresolved value),
5. require explicit acknowledgment for every needs-create group,
6. write inside ONE transaction, stamping the batch id onto the journal:
a. create each acknowledged needs-create business unit (journal insert),
b. update each in-scope asset's name / location / business unit
(journal update, capturing the pre-image first).
Everything commits together or not at all. Ambiguous rows (one workstation key
matching several machines) are never written - the dry-run flags them and the
writer skips them, because guessing which machine an export row means is exactly
the kind of cleverness this import refuses.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))))
from dotenv import load_dotenv
from pathlib import Path
load_dotenv(Path(__file__).resolve().parents[3] / '.env')
from scripts.site_imports import journal
from scripts.site_imports.approval import build_packet, packet_matches_plan
from scripts.site_imports.resolver import NEEDS_CREATE, RESOLVED, norm
from scripts.site_imports.winc.dry_run import (
analyze,
build_app_connection,
)
# The packet groups the WINC dry-run declares, reused verbatim so the writer
# hashes and reads the plan exactly the way approval did.
_PACKET_GROUPS = [
('businessunit', 'businessunit_status', 'raw_eq1', 'businessunit_name'),
('location', 'location_status', 'raw_location', 'location_name'),
]
# References into a created row's id, so undo can tell whether an inserted
# business unit is still pointed at by something outside this batch before it
# hard-deletes it. Shared with the undo CLI and the web undo path.
_UNDO_REFERENCES = {
'businessunits': [('assets', 'businessunitid')],
}
# References into a created row's id, so undo can tell whether an inserted
# business unit is still pointed at by something outside this batch before it
# hard-deletes it.
# Max width of assets.name; the export description can exceed it (a few rows
# carry a whole safety notice in the field), so a set name is clamped rather
# than letting the whole transaction fail on one over-long row. The pre-image is
# journalled, so the clamp is reversible like any other change.
_ASSET_NAME_MAXLEN = 100
class WriterError(Exception):
"""The plan cannot be written as approved (drift, blocking, or missing
acknowledgment). Carries a human-readable reason; never a partial write."""
def _load_asset_ids(connection):
"""assetnumber -> assetid for every asset, so a plan row (which carries the
matched asset *number*) can be turned into the id the update targets."""
from sqlalchemy import text
out = {}
for assetid, assetnumber in connection.execute(text(
'SELECT assetid, assetnumber FROM assets')):
out[assetnumber] = assetid
return out
def _create_businessunit(connection, batchid, sequence, name):
"""Insert one business unit and journal it. Returns (new_id, next_sequence).
Only ever called for an acknowledged needs-create name, so the journal's
insert entry describes strictly a row this batch created."""
from sqlalchemy import text
now = journal._utcnow()
result = connection.execute(text(
'INSERT INTO businessunits '
'(businessunit, createddate, modifieddate, isactive) '
'VALUES (:name, :now, :now, 1)'),
{'name': name, 'now': now})
new_id = result.lastrowid
journal.record_insert(connection, batchid, sequence, 'businessunits',
new_id)
return new_id, sequence + 1
def _update_cell(connection, batchid, sequence, table, targetid, column,
oldvalue, newvalue):
"""Apply one cell update and journal its pre/post image. Returns the next
sequence. Skips (and does not journal) a no-op where the value is unchanged."""
from sqlalchemy import text
if journal._as_text(oldvalue) == journal._as_text(newvalue):
return sequence
connection.execute(text(
'UPDATE %s SET %s = :newvalue WHERE %s = :targetid'
% (table, column, _asset_pk(table))),
{'newvalue': newvalue, 'targetid': targetid})
journal.record_update(connection, batchid, sequence, table, targetid,
column, oldvalue, newvalue)
return sequence + 1
def _asset_pk(table):
# The writer only updates the assets table; keep the mapping explicit rather
# than reflecting, so an unexpected table name fails loudly.
return {'assets': 'assetid'}[table]
def apply_plan(connection, plan, approved_packet, acknowledge=None,
createdby=None):
"""Apply an approved WINC plan, reversibly. Returns the batch id.
connection a live SQLAlchemy connection in an open transaction.
plan the per-row plan from dry_run.analyze (rebuilt fresh).
approved_packet the packet a human signed off on.
acknowledge set of needs-create group names the human confirmed
(e.g. {'businessunit'}); required for every group the
packet lists under needscreate.
createdby who acknowledged, stamped on the batch.
Raises WriterError before touching anything if the plan drifted, is
blocking, or a needs-create group was not acknowledged.
"""
acknowledge = set(acknowledge or ())
# (2)-(3) rebuild the packet and prove nothing drifted since approval.
current_packet = build_packet('winc-machine-adoption', plan,
approved_packet.get('counts', {}),
_PACKET_GROUPS)
if not packet_matches_plan(approved_packet, plan):
raise WriterError(
'plan has changed since approval (hash mismatch) - re-run the '
'dry-run and re-approve before importing')
# (4) never write over an unresolved value.
if current_packet.get('blocking'):
raise WriterError(
'plan is blocking: unresolved values remain - add the missing '
'aliases and re-run the dry-run')
# (5) every needs-create group must be explicitly acknowledged.
missing_ack = set(current_packet.get('needscreate', {})) - acknowledge
if missing_ack:
raise WriterError(
'needs-create not acknowledged for: %s - confirm these will be '
'created before importing' % ', '.join(sorted(missing_ack)))
batchid = approved_packet['batchid']
journal.open_batch(connection, approved_packet, createdby=createdby)
sequence = 1
# (6a) create acknowledged needs-create business units once each, first, so
# the asset updates below can point at them.
created_bu = {}
for row in plan:
if row['businessunit_status'] != NEEDS_CREATE:
continue
name = row['businessunit_name']
key = norm(name)
if key not in created_bu:
created_bu[key], sequence = _create_businessunit(
connection, batchid, sequence, name)
# (6b) update each in-scope, unambiguous asset.
asset_ids = _load_asset_ids(connection)
from sqlalchemy import text
for row in plan:
if row['ambiguous']:
continue
assetnumber = row['matched_asset']
assetid = asset_ids.get(assetnumber)
if assetid is None:
continue
current = connection.execute(text(
'SELECT name, locationid, businessunitid FROM assets '
'WHERE assetid = :assetid'), {'assetid': assetid}).fetchone()
cur_name, cur_location, cur_bu = current
if row['name']:
new_name = row['name'][:_ASSET_NAME_MAXLEN]
sequence = _update_cell(connection, batchid, sequence, 'assets',
assetid, 'name', cur_name, new_name)
if row['location_status'] == RESOLVED and row['locationid'] != '':
sequence = _update_cell(connection, batchid, sequence, 'assets',
assetid, 'locationid', cur_location,
row['locationid'])
new_bu = None
if row['businessunit_status'] == RESOLVED and row['businessunitid'] != '':
new_bu = row['businessunitid']
elif row['businessunit_status'] == NEEDS_CREATE:
new_bu = created_bu[norm(row['businessunit_name'])]
if new_bu is not None:
sequence = _update_cell(connection, batchid, sequence, 'assets',
assetid, 'businessunitid', cur_bu, new_bu)
return batchid
def main(argv=None):
parser = argparse.ArgumentParser(
description='WINC machine adoption writer. Applies an approved plan '
'reversibly. Requires --commit to write.')
parser.add_argument('--export', required=True,
help='Path to the ASSET export CSV.')
parser.add_argument('--acknowledge', action='append', default=[],
metavar='GROUP',
help='Acknowledge a needs-create group (repeatable), '
'e.g. --acknowledge businessunit.')
parser.add_argument('--createdby', default=None,
help='Who is approving this import (stamped on batch).')
parser.add_argument('--commit', action='store_true',
help='Actually write. Without it, this is a no-op that '
'reports what would happen.')
args = parser.parse_args(argv)
if not os.path.exists(args.export):
parser.error('export file not found: %s' % args.export)
_app, connection = build_app_connection()
plan, summary, _lookups = analyze(connection, args.export)
packet = build_packet('winc-machine-adoption', plan, summary,
_PACKET_GROUPS)
if not args.commit:
print('DRY: rebuilt plan of %d rows, hash %s' % (
packet['rowcount'], packet['planhash']))
print('DRY: pass --commit with --acknowledge for each needs-create '
'group to write. Needs-create groups: %s'
% (', '.join(packet.get('needscreate', {})) or 'none'))
return 0
from shopdb.api import db
try:
batchid = apply_plan(connection, plan, packet,
acknowledge=args.acknowledge,
createdby=args.createdby)
db.session.commit()
except WriterError as error:
db.session.rollback()
print('REFUSED: %s' % error)
return 2
except Exception:
db.session.rollback()
raise
print('COMMITTED batch %s' % batchid)
print(' to undo: python -m scripts.site_imports.winc.undo --batch %s'
% batchid)
return 0
if __name__ == '__main__':
sys.exit(main())