From 2998e35ddf685f7499828535922abfc3437a8837 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 05:04:22 -0400 Subject: [PATCH] Add reversible Maximo/WINC machine-adoption import Machine data arrives as a Maximo asset export (validated against the WINC site) that must update existing machines in place, not create canonical per-table rows, so it does not fit the ShopDB-native/Azure/Thor CSV adapter path. Bulk edits to live machines also need an escape hatch, so the whole commit is journaled and can be undone as one batch. Add a dedicated dry-run analyzer, approval packet, resolver/suggest helpers, and a reversible writer/undo backed by a new importbatches/importjournalentries schema (with migration). Expose preview/commit/batches/undo settings routes and a matching frontend source card and panel, and surface the batch/journal contract through shopdb.api for plugins. --- docs/PLUGIN-HOOKS.md | 45 ++ frontend/src/api/index.js | 30 ++ .../views/settings/ImportSiteAssets.spec.js | 187 +++++++++ .../src/views/settings/ImportSiteAssets.vue | 390 +++++++++++++++++- .../7d37_import_batches_and_journal.py | 89 ++++ scripts/site_imports/approval.py | 149 +++++++ scripts/site_imports/clone_import_db.ps1 | 51 +++ scripts/site_imports/journal.py | 267 ++++++++++++ scripts/site_imports/resolver.py | 134 ++++++ scripts/site_imports/suggest.py | 159 +++++++ scripts/site_imports/winc/__init__.py | 0 scripts/site_imports/winc/aliases.py | 67 +++ scripts/site_imports/winc/dry_run.py | 385 +++++++++++++++++ scripts/site_imports/winc/undo.py | 55 +++ scripts/site_imports/winc/writer.py | 278 +++++++++++++ shopdb/__init__.py | 8 +- shopdb/api/__init__.py | 16 + shopdb/core/api/settings.py | 117 ++++++ shopdb/core/models/__init__.py | 9 + shopdb/core/models/importbatch.py | 86 ++++ shopdb/core/services/wincimport.py | 138 +++++++ tests/test_winc_import_routes.py | 210 ++++++++++ 22 files changed, 2859 insertions(+), 11 deletions(-) create mode 100644 docs/PLUGIN-HOOKS.md create mode 100644 frontend/src/views/settings/ImportSiteAssets.spec.js create mode 100644 migrations/versions/7d37_import_batches_and_journal.py create mode 100644 scripts/site_imports/approval.py create mode 100644 scripts/site_imports/clone_import_db.ps1 create mode 100644 scripts/site_imports/journal.py create mode 100644 scripts/site_imports/resolver.py create mode 100644 scripts/site_imports/suggest.py create mode 100644 scripts/site_imports/winc/__init__.py create mode 100644 scripts/site_imports/winc/aliases.py create mode 100644 scripts/site_imports/winc/dry_run.py create mode 100644 scripts/site_imports/winc/undo.py create mode 100644 scripts/site_imports/winc/writer.py create mode 100644 shopdb/core/models/importbatch.py create mode 100644 shopdb/core/services/wincimport.py create mode 100644 tests/test_winc_import_routes.py diff --git a/docs/PLUGIN-HOOKS.md b/docs/PLUGIN-HOOKS.md new file mode 100644 index 00000000..a4b3f531 --- /dev/null +++ b/docs/PLUGIN-HOOKS.md @@ -0,0 +1,45 @@ +# Plugin hooks and contract surface + +Everything a plugin may import from the core lives in `shopdb.api` (ADR-001), +versioned by `__contract_version__` (ADR-002). Adding a name is a minor bump; +removing one is major. This file records the non-obvious hooks - the ones a +plugin author would not guess from the name alone. It is not the full export +list; read `shopdb/api/__init__.py` for that. + +## Reversible bulk-import journal + +Contract version 0.30.0. + +A plugin that drives a bulk import - a site asset adoption, an integration sync - +records the primitive changes it makes so the whole batch can be reversed later. +The journal is import-agnostic: it stores primitive mutations (set a cell, insert +a row) tied to a batch id, and a generic undo replays them in reverse. Nothing in +core learns what the import means. + +Exposed on `shopdb.api`: + +- `ImportBatch` - one row per applied import. `batchid` is the identity that the + approval step, the write, and the undo all reference. `status` is `applied` or + `undone`. +- `ImportJournalEntry` - one reversible primitive mutation. `operation` is + `update` (a cell change, carrying `oldvalue`/`newvalue` so it can be put back) + or `insert` (a row this batch created, so undo can delete strictly what the + import created). +- `BATCH_APPLIED`, `BATCH_UNDONE` - the two `ImportBatch.status` values. +- `OP_UPDATE`, `OP_INSERT` - the two `ImportJournalEntry.operation` values. + +What a plugin gets from this, and what it does NOT: + +- A plugin's import UI reads `ImportBatch` to list applied batches (kind, row + count, who applied it, when) and offers an "Undo import" per batch. +- A plugin does NOT write the journal or run undo through these model imports. + The write/undo logic lives with the import writer (see + `scripts/site_imports/journal.py`), which records entries in the SAME + transaction as the mutation they describe and reverses them conservatively: + an `update` is put back only if the cell still holds the value the import + wrote (a value a human changed since is left as-is and reported), and an + `insert` is hard-deleted only when nothing outside the batch still references + the created row. + +The tables are core-owned (migration `7d37_import_batches_and_journal`), so they +exist regardless of which plugins are installed. diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 57fcc970..304ece15 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1204,6 +1204,36 @@ export const settingsApi = { return api.post('/settings/asset-import/import', form, { headers: { 'Content-Type': 'multipart/form-data' } }) + }, + wincPreview(file) { + // Read-only WINC (Maximo-style) machine-adoption analysis. Uploads one CSV + // and returns the approval packet (counts, needs-create business units, + // unresolved values); writes nothing. + const form = new FormData() + form.append('files', file) + return api.post('/settings/asset-import/winc/preview', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) + }, + wincCommit(file, acknowledge = []) { + // Apply a WINC plan reversibly. acknowledge is the list of needs-create + // group names (e.g. ['businessunit']) the admin confirmed. + const form = new FormData() + form.append('files', file) + if (acknowledge.length) { + form.append('acknowledge', acknowledge.join(',')) + } + return api.post('/settings/asset-import/winc/commit', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) + }, + wincBatches() { + // Applied/undone WINC import batches, newest first, for the undo list. + return api.get('/settings/asset-import/winc/batches') + }, + wincUndo(batchid) { + // Reverse an applied WINC import batch by id. + return api.post('/settings/asset-import/winc/undo', { batchid }) } } diff --git a/frontend/src/views/settings/ImportSiteAssets.spec.js b/frontend/src/views/settings/ImportSiteAssets.spec.js new file mode 100644 index 00000000..a682a15b --- /dev/null +++ b/frontend/src/views/settings/ImportSiteAssets.spec.js @@ -0,0 +1,187 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { mount, flushPromises } from '@vue/test-utils' + +// The WINC source is the one import path that is NOT a canonical-CSV adapter: +// it previews a machine-adoption plan, gates a commit behind an explicit +// business-unit acknowledgment, and exposes an undo list. These specs pin that +// gate and the preview/commit/undo wiring, mocking the settings API so no +// network or backend is involved. + +vi.mock('@/api', () => ({ + settingsApi: { + get: vi.fn().mockRejectedValue({ response: { status: 404 } }), + listImportTemplates: vi.fn().mockResolvedValue({ data: { data: { tables: [] } } }), + wincPreview: vi.fn(), + wincCommit: vi.fn(), + wincBatches: vi.fn().mockResolvedValue({ data: { data: { batches: [] } } }), + wincUndo: vi.fn().mockResolvedValue({ data: { data: { reverted: 3, conflicts: [] } } }), + }, + computersApi: { types: { listAll: vi.fn().mockResolvedValue([]) } }, +})) +vi.mock('@/utils/apiError', () => ({ apiError: (e) => String(e) })) +vi.mock('../../utils/apiError', () => ({ apiError: (e) => String(e) })) + +import { settingsApi } from '@/api' +import ImportSiteAssets from './ImportSiteAssets.vue' + +function previewReport(overrides = {}) { + return { + data: { + data: { + committed: false, + blocking: false, + rowcount: 1, + summary: { in_scope: 1, skipped_no_existing_machine: 0, ambiguous_match: 0 }, + needscreate: { businessunit: { 'Large Parts': 1 } }, + unresolved: {}, + ...overrides, + }, + }, + } +} + +async function render() { + const wrapper = mount(ImportSiteAssets, { + global: { + stubs: { 'router-link': { template: '' } }, + }, + }) + await flushPromises() + return wrapper +} + +function selectWinc(wrapper) { + // The WINC card is the fourth source-format button. + const card = wrapper.findAll('.source-card').find(c => c.text().includes('WINC')) + return card.trigger('click') +} + +async function pickFile(wrapper) { + const input = wrapper.find('.winc-panel input[type="file"]') + const file = new File(['assetnum\nW0083A\n'], 'winc.csv', { type: 'text/csv' }) + Object.defineProperty(input.element, 'files', { value: [file], configurable: true }) + await input.trigger('change') + return file +} + +describe('WINC source selection', () => { + beforeEach(() => { + settingsApi.wincPreview.mockReset() + settingsApi.wincCommit.mockReset() + }) + + it('offers WINC as a fourth source card', async () => { + const wrapper = await render() + const labels = wrapper.findAll('.source-card').map(c => c.text()) + expect(labels.some(t => t.includes('WINC'))).toBe(true) + }) + + it('shows the WINC panel instead of the generic controls when selected', async () => { + const wrapper = await render() + await selectWinc(wrapper) + expect(wrapper.find('.winc-panel').exists()).toBe(true) + }) +}) + +describe('WINC preview and commit gating', () => { + beforeEach(() => { + settingsApi.wincPreview.mockReset() + settingsApi.wincCommit.mockReset() + settingsApi.wincBatches.mockClear() + settingsApi.wincBatches.mockResolvedValue({ data: { data: { batches: [] } } }) + }) + + it('previews the uploaded export and lists needs-create business units', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport()) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + expect(settingsApi.wincPreview).toHaveBeenCalledOnce() + expect(wrapper.find('.winc-needscreate').text()).toContain('Large Parts') + }) + + it('blocks commit until the business-unit creation is acknowledged', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport()) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + const commitBtn = wrapper.find('.winc-panel .btn-primary') + expect(commitBtn.attributes('disabled')).toBeDefined() + + await wrapper.find('.winc-ack input[type="checkbox"]').setValue(true) + expect(commitBtn.attributes('disabled')).toBeUndefined() + }) + + it('commits with the businessunit acknowledgment and refreshes the batch list', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport()) + settingsApi.wincCommit.mockResolvedValue({ + data: { data: { committed: true, rowcount: 1, batchid: 'b-1', summary: { in_scope: 1, skipped_no_existing_machine: 0, ambiguous_match: 0 }, needscreate: {}, unresolved: {} } }, + }) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + await wrapper.find('.winc-ack input[type="checkbox"]').setValue(true) + + await wrapper.find('.winc-panel .btn-primary').trigger('click') + await flushPromises() + + const [file, acknowledge] = settingsApi.wincCommit.mock.calls[0] + expect(file.name).toBe('winc.csv') + expect(acknowledge).toEqual(['businessunit']) + // Batches reloaded once on mount and once after commit. + expect(settingsApi.wincBatches).toHaveBeenCalledTimes(2) + }) + + it('does not require acknowledgment when there is nothing to create', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport({ needscreate: {} })) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + expect(wrapper.find('.winc-ack').exists()).toBe(false) + expect(wrapper.find('.winc-panel .btn-primary').attributes('disabled')).toBeUndefined() + }) + + it('blocks commit when the plan is blocking', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport({ + blocking: true, needscreate: {}, unresolved: { location: { 'WM-XYZ': 2 } }, + })) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + expect(wrapper.find('.winc-panel .btn-primary').attributes('disabled')).toBeDefined() + expect(wrapper.find('.problems').text()).toContain('WM-XYZ') + }) +}) + +describe('WINC undo', () => { + it('undoes an applied batch and reloads the list', async () => { + settingsApi.wincBatches.mockResolvedValue({ + data: { data: { batches: [ + { batchid: 'b-1', kind: 'winc-machine-adoption', rowcount: 5, createdat: '2026-01-01T00:00:00Z', createdby: 'admin', status: 'applied', undoneat: null }, + ] } }, + }) + const wrapper = await render() + await selectWinc(wrapper) + await flushPromises() + + await wrapper.find('.winc-batches .btn-secondary').trigger('click') + await flushPromises() + + expect(settingsApi.wincUndo).toHaveBeenCalledWith('b-1') + }) +}) diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue index 679eedc1..445e0d84 100644 --- a/frontend/src/views/settings/ImportSiteAssets.vue +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -112,6 +112,15 @@ :class="['source-card', { active: importSource === s.key }]" @click="selectSource(s.key)" > + + + {{ s.tag }} + + {{ s.label }} {{ s.description }} @@ -204,7 +213,166 @@

-
+
+

+ A WINC (Maximo-style) asset export adopts existing machines: it never + creates an asset, only updates a machine that already exists, matched + on its workstation number. It sets the machine's name, location, and + business unit. Preview first - it writes nothing and reports what would + change, which business units it would create (you must confirm), and + any values it cannot resolve (which block the import). Every commit is + journalled, so it can be undone from the list below. +

+ +
+ +
+ + + +
+
+ +

Selected: {{ wincFile.name }}

+

{{ wincError }}

+ +
+

+ + + +

+ + + + + + + + + + + + + + + + +
In scope (existing machines matched){{ wincReport.summary.in_scope }}
Skipped (no existing machine){{ wincReport.summary.skipped_no_existing_machine }}
Ambiguous (skipped, not guessed){{ wincReport.summary.ambiguous_match }}
+ +
+

New business units to be created

+

+ These business units are referenced by the export but do not exist + yet. Confirm they are correct - you can edit them later in + Settings. Nothing is created until you acknowledge and commit. +

+
    +
  • + {{ name }} ({{ count }} machine(s)) +
    • +
+ +
+ +
+

Unresolved values (blocking)

+

+ These export values have no mapping yet, so the import will not + touch the affected rows. An administrator must add an alias for + each before this import can run. +

+
    +
  • + {{ group }}: + + "{{ value }}" ({{ count }}){{ ' ' }} + +
    • +
+
+
+ +
+

Import history

+

+ Each committed WINC import can be undone. Undo reverses only cells + this import set that have not been changed since, and removes only + business units this import created. +

+ + + + + + + + + + + + + + + + + + + +
WhenByRowsStatus
{{ b.createdat }}{{ b.createdby || '-' }}{{ b.rowcount }}{{ b.status }} + + undone +
+
+
+ +
+ !!(wincReport.value && wincReport.value.needscreate + && wincReport.value.needscreate.businessunit)) + +const wincCanCommit = computed(() => { + const wincResult = wincReport.value + if (!wincResult || wincCommitting.value || wincPreviewing.value) return false + if (wincResult.committed || wincResult.blocking) return false + if (wincResult.rowcount === 0) return false + if (wincNeedsBusinessUnit.value && !wincAckBusinessUnit.value) return false + return true +}) + // Commit is allowed only after a clean preview of the CURRENT selection: a // report with no problems and no request-level error that has not already been // committed. @@ -651,6 +884,77 @@ async function commit() { } } +function onWincFilePicked(event) { + wincFile.value = (event.target.files || [])[0] || null + // A new file invalidates the previous report and its acknowledgment. + wincReport.value = null + wincError.value = '' + wincAckBusinessUnit.value = false +} + +function clearWincSelection() { + wincFile.value = null + wincReport.value = null + wincError.value = '' + wincAckBusinessUnit.value = false + if (wincFileInput.value) wincFileInput.value.value = '' +} + +async function wincPreview() { + if (!wincFile.value) return + wincPreviewing.value = true + wincError.value = '' + wincAckBusinessUnit.value = false + try { + const response = await settingsApi.wincPreview(wincFile.value) + wincReport.value = response.data?.data || response.data || null + } catch (err) { + wincError.value = apiError(err, 'WINC preview failed.') + wincReport.value = null + } finally { + wincPreviewing.value = false + } +} + +async function wincCommit() { + if (!wincCanCommit.value) return + wincCommitting.value = true + wincError.value = '' + try { + const acknowledge = wincNeedsBusinessUnit.value ? ['businessunit'] : [] + const response = await settingsApi.wincCommit(wincFile.value, acknowledge) + wincReport.value = response.data?.data || response.data || null + await loadWincBatches() + } catch (err) { + wincError.value = apiError(err, 'WINC import failed.') + } finally { + wincCommitting.value = false + } +} + +async function loadWincBatches() { + try { + const response = await settingsApi.wincBatches() + const data = response.data?.data || response.data || {} + wincBatches.value = data.batches || [] + } catch { + wincBatches.value = [] + } +} + +async function undoWincBatch(batchid) { + wincUndoingId.value = batchid + wincError.value = '' + try { + await settingsApi.wincUndo(batchid) + await loadWincBatches() + } catch (err) { + wincError.value = apiError(err, 'WINC undo failed.') + } finally { + wincUndoingId.value = '' + } +} + async function addComputerType() { const name = customComputerType.value.trim() if (!name) return @@ -683,6 +987,7 @@ onMounted(async () => { await loadLocation() await loadTemplates() await loadComputerTypes() + await loadWincBatches() }) @@ -769,27 +1074,63 @@ onMounted(async () => { .source-cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); - gap: 10px; - margin: 4px 0; + gap: 12px; + margin: 6px 0; } .source-card { display: flex; flex-direction: column; - gap: 4px; + gap: 6px; text-align: left; - padding: 12px 14px; + padding: 14px; border: 1px solid var(--border); - border-radius: 8px; + border-radius: 10px; background: var(--bg); color: var(--text); cursor: pointer; + position: relative; + transition: border-color 0.15s ease, box-shadow 0.15s ease, transform 0.15s ease, background 0.15s ease; } .source-card:hover { border-color: var(--primary); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.18); + transform: translateY(-1px); } .source-card.active { border-color: var(--primary); - box-shadow: 0 0 0 1px var(--primary) inset; + box-shadow: 0 0 0 1px var(--primary) inset, 0 2px 10px rgba(0, 0, 0, 0.22); + background: color-mix(in srgb, var(--primary) 8%, var(--bg)); +} +.source-card-head { + display: flex; + align-items: center; + gap: 8px; +} +.source-card-icon { + width: 22px; + height: 22px; + color: var(--text-light); + flex: none; +} +.source-card.active .source-card-icon { + color: var(--primary); +} +.source-card-tag { + font-size: 0.68rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + padding: 2px 8px; + border-radius: 999px; + background: color-mix(in srgb, var(--primary) 14%, transparent); + color: var(--primary); +} +.source-card-check { + width: 18px; + height: 18px; + color: var(--primary); + margin-left: auto; + flex: none; } .source-card-title { font-weight: 600; @@ -963,4 +1304,33 @@ onMounted(async () => { margin: 4px 0; font-size: 0.9rem; } +.winc-panel { + margin-top: 12px; +} +.winc-needscreate { + margin-top: 12px; + border: 1px solid var(--warning, #d0a44a); + border-radius: 8px; + padding: 12px 14px; +} +.winc-needscreate h4 { + margin: 0 0 6px; +} +.winc-needscreate ul { + margin: 6px 0; + padding-left: 20px; +} +.winc-ack { + display: flex; + align-items: center; + gap: 8px; + margin-top: 8px; + font-size: 0.9rem; +} +.winc-batches { + margin-top: 16px; +} +.winc-batches h4 { + margin: 0 0 6px; +} diff --git a/migrations/versions/7d37_import_batches_and_journal.py b/migrations/versions/7d37_import_batches_and_journal.py new file mode 100644 index 00000000..daf99fd1 --- /dev/null +++ b/migrations/versions/7d37_import_batches_and_journal.py @@ -0,0 +1,89 @@ +"""Add importbatches and importjournalentries (reversible bulk imports) + +Two additive, platform-level tables that let any bulk-import writer record what +it changed so the change can be reversed later. Import-agnostic: an importer +writes primitive mutations (update a cell, insert a row) tied to a batch id, and +a generic undo replays them in reverse. + + importbatches one row per approved write (identity: batchid, the same + value the approval packet carries). + importjournalentries one row per reversible primitive mutation, ordered by + sequence so undo can walk them in reverse. + +Idempotent guards and a real downgrade throughout; strict MySQL 8 clean. + +Revision ID: 7d37_import_batches_and_journal +Revises: 7d36_scriptlibrary_and_app_installer_digest +Create Date: 2026-09-03 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = '7d37_import_batches_and_journal' +down_revision = '7d36_scriptlibrary_and_app_installer_digest' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + tables = insp.get_table_names() + + if 'importbatches' not in tables: + op.create_table( + 'importbatches', + sa.Column('batchid', sa.String(length=36), primary_key=True), + sa.Column('kind', sa.String(length=64), nullable=False), + sa.Column('planhash', sa.String(length=64), nullable=False), + sa.Column('rowcount', sa.Integer(), nullable=False, + server_default='0'), + sa.Column('createdat', sa.DateTime(), nullable=False), + sa.Column('createdby', sa.String(length=100), nullable=True), + sa.Column('status', sa.String(length=16), nullable=False, + server_default='applied'), + sa.Column('undoneat', sa.DateTime(), nullable=True), + ) + + if 'importjournalentries' not in tables: + op.create_table( + 'importjournalentries', + sa.Column('entryid', sa.BigInteger(), primary_key=True, + autoincrement=True), + sa.Column('batchid', sa.String(length=36), nullable=False), + sa.Column('sequence', sa.Integer(), nullable=False), + sa.Column('operation', sa.String(length=16), nullable=False), + sa.Column('targettable', sa.String(length=64), nullable=False), + sa.Column('targetid', sa.BigInteger(), nullable=False), + sa.Column('columnname', sa.String(length=64), nullable=True), + sa.Column('oldvalue', sa.Text(), nullable=True), + sa.Column('newvalue', sa.Text(), nullable=True), + sa.Column('isreverted', sa.Boolean(), nullable=False, + server_default=sa.text('0')), + sa.ForeignKeyConstraint( + ['batchid'], ['importbatches.batchid'], + name='fk_importjournalentries_batchid', + ondelete='CASCADE'), + ) + op.create_index( + 'idx_importjournalentries_batchid', + 'importjournalentries', ['batchid']) + + +def downgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + tables = insp.get_table_names() + + if 'importjournalentries' in tables: + existing = [ix['name'] for ix in + insp.get_indexes('importjournalentries')] + if 'idx_importjournalentries_batchid' in existing: + op.drop_index('idx_importjournalentries_batchid', + table_name='importjournalentries') + op.drop_table('importjournalentries') + + if 'importbatches' in tables: + op.drop_table('importbatches') diff --git a/scripts/site_imports/approval.py b/scripts/site_imports/approval.py new file mode 100644 index 00000000..5892f867 --- /dev/null +++ b/scripts/site_imports/approval.py @@ -0,0 +1,149 @@ +"""Reusable approval / promotion gate for site-import writers. + +IMPORT-AGNOSTIC. This turns a dry-run plan into an "approval packet": a frozen +summary of what an import WOULD do, plus a content hash of the exact plan it +describes. A writer refuses to run unless it is handed a packet whose hash still +matches the plan in front of it. That is the whole mechanism: a human approves a +specific, frozen plan, and the writer can prove nothing drifted since. + +The same packet shape serves every importer, and it is deliberately the seam the +undo work will reuse: a writer records the packet's batch id alongside the +journal it writes, so approval, write, and undo all reference one identity. + +A packet is a plain dict (JSON-serialisable) so it can be printed, saved next to +an export, diffed between runs, and shown in a UI without any of this code: + + { + 'kind': caller-supplied label, e.g. 'winc-machine-adoption' + 'planhash': sha256 of the canonicalised plan rows + 'rowcount': number of rows in the plan + 'counts': caller-supplied summary dict (what the report shows) + 'needscreate': { group -> { name -> count } } # requires acknowledgment + 'unresolved': { group -> { value -> count } } # blocks approval + 'blocking': True if any unresolved entries exist + } + +The writer's contract, in order: + 1. rebuild the plan from the same inputs, + 2. build a packet from it, + 3. compare planhash to the approved packet - abort on mismatch, + 4. refuse if packet['blocking'], + 5. require explicit acknowledgment for every needscreate group, + 6. only then write, stamping packet['batchid'] onto the journal. +""" + +import hashlib +import json +import uuid +from collections import OrderedDict + +from scripts.site_imports.resolver import tally, NEEDS_CREATE, UNRESOLVED + + +def plan_hash(plan): + """Stable sha256 over the plan's rows. + + Canonicalised by serialising each row as sorted-key JSON so the hash depends + only on content, not on dict ordering or whitespace. Two runs that produce + the same decisions produce the same hash; any changed decision changes it, + which is exactly what the writer checks before it touches the database. + """ + hasher = hashlib.sha256() + for row in plan: + canonical = json.dumps(row, sort_keys=True, default=str, + separators=(',', ':')) + hasher.update(canonical.encode('utf-8')) + hasher.update(b'\n') + return hasher.hexdigest() + + +def build_packet(kind, plan, counts, groups): + """Build an approval packet from a plan. + + kind a short label identifying the importer, stored on the packet. + plan the list of per-row dicts the writer will act on. + counts the caller's summary dict (shown to the human as-is). + groups how to read each field's status/value columns out of a row: + [ (group_name, status_column, value_column), ... ] + e.g. ('businessunit', 'businessunit_status', 'raw_eq1'). The value + column is what a human would act on - the raw export value for + UNRESOLVED (so they know what to alias) and the canonical name for + NEEDS_CREATE (so they know what would be created). + + needscreate is keyed on the canonical name column when present; callers pass + the *_name column for that via the fourth tuple element when it differs from + the value column. + """ + needscreate = OrderedDict() + unresolved = OrderedDict() + for group in groups: + group_name, status_column, value_column = group[0], group[1], group[2] + name_column = group[3] if len(group) > 3 else value_column + unresolved_hits = tally(plan, status_column, value_column, UNRESOLVED) + needs_hits = tally(plan, status_column, name_column, NEEDS_CREATE) + if unresolved_hits: + unresolved[group_name] = unresolved_hits + if needs_hits: + needscreate[group_name] = needs_hits + + return OrderedDict([ + ('kind', kind), + ('batchid', str(uuid.uuid4())), + ('planhash', plan_hash(plan)), + ('rowcount', len(plan)), + ('counts', OrderedDict(counts)), + ('needscreate', needscreate), + ('unresolved', unresolved), + ('blocking', bool(unresolved)), + ]) + + +def packet_matches_plan(packet, plan): + """True if the approved packet still describes this exact plan.""" + return packet.get('planhash') == plan_hash(plan) + + +def format_packet(packet): + """Human-readable rendering of a packet for the console or a UI panel.""" + lines = [] + lines.append('APPROVAL PACKET: %s' % packet.get('kind', '?')) + lines.append(' batch id : %s' % packet.get('batchid', '?')) + lines.append(' plan hash: %s' % packet.get('planhash', '?')) + lines.append(' rows : %d' % packet.get('rowcount', 0)) + + needscreate = packet.get('needscreate') or {} + if needscreate: + for group, items in needscreate.items(): + total = sum(items.values()) + lines.append('') + lines.append(' WARNING: this import would CREATE %d %s(s). ' + 'Confirm these are correct; you can edit them later ' + 'in Settings:' % (len(items), group)) + for name, count in items.items(): + lines.append(' + %-28s (%d rows)' % (name, count)) + lines.append(' total new %s rows: %d' % (group, total)) + + unresolved = packet.get('unresolved') or {} + if unresolved: + for group, items in unresolved.items(): + lines.append('') + lines.append(' BLOCKING: %d unresolved %s value(s) - add an alias ' + 'for each before importing:' % (len(items), group)) + for value, count in items.items(): + lines.append(' ? %-28s (%d rows)' % (value, count)) + + lines.append('') + lines.append(' status : %s' % ( + 'BLOCKED (resolve the values above)' if packet.get('blocking') + else 'READY for acknowledgment')) + return '\n'.join(lines) + + +def save_packet(packet, path): + with open(path, 'w', encoding='utf-8') as handle: + json.dump(packet, handle, indent=2) + + +def load_packet(path): + with open(path, encoding='utf-8') as handle: + return json.load(handle) diff --git a/scripts/site_imports/clone_import_db.ps1 b/scripts/site_imports/clone_import_db.ps1 new file mode 100644 index 00000000..5f7c4de0 --- /dev/null +++ b/scripts/site_imports/clone_import_db.ps1 @@ -0,0 +1,51 @@ +# Clone the live shopdb_flask database into a throwaway copy for import testing. +# +# Runs entirely inside the db container so the root password is read from the +# container's own environment and never typed on a command line or logged. The +# clone is a full structural + data copy, so a dry run or a writer pointed at it +# behaves exactly as it would against production - but nothing it does can touch +# the real data. +# +# pwsh scripts/site_imports/clone_import_db.ps1 +# pwsh scripts/site_imports/clone_import_db.ps1 -Target shopdb_flask_import +# +# Point a tool at the clone by overriding DATABASE_URL for that one run, e.g. +# docker compose exec -e DATABASE_URL="mysql+pymysql://root:PW@db:3306/shopdb_flask_import?charset=utf8mb4" api ... +# (the winc dry run also honours DATABASE_URL, so --export works unchanged). + +param( + [string]$Source = 'shopdb_flask', + [string]$Target = 'shopdb_flask_import' +) + +$ErrorActionPreference = 'Stop' + +Write-Host "Cloning '$Source' -> '$Target' inside the db container..." + +# All work happens in the container. $MYSQL_ROOT_PASSWORD is already set there. +# MYSQL_PWD is honoured by both mysql and mysqldump, which keeps the password +# off the argument list (no -p on the command line to leak into ps or logs). +# Backticks around identifiers are MySQL's; placeholders are filled in by +# PowerShell below so the shell here sees plain literal database names. +$remote = @' +set -e +export MYSQL_PWD="$MYSQL_ROOT_PASSWORD" +mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS \`__TARGET__\`; CREATE DATABASE \`__TARGET__\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" +mysqldump -h 127.0.0.1 -u root --single-transaction --routines --triggers __SOURCE__ | mysql -h 127.0.0.1 -u root __TARGET__ +mysql -h 127.0.0.1 -u root -N -e "SELECT CONCAT('assets in clone: ', COUNT(*)) FROM __TARGET__.assets;" +'@ + +$remote = $remote.Replace('__TARGET__', $Target).Replace('__SOURCE__', $Source) + +# Strip CR so the container's sh sees clean LF-only lines (this file is CRLF on +# a Windows checkout, and sh treats a trailing \r as part of the argument). +$remote = $remote.Replace("`r", '') + +# The db service name in docker-compose.yml is 'db'. +docker compose exec -T db sh -c $remote + +if ($LASTEXITCODE -ne 0) { + throw "Clone failed (exit $LASTEXITCODE)." +} + +Write-Host "Clone complete: $Target" diff --git a/scripts/site_imports/journal.py b/scripts/site_imports/journal.py new file mode 100644 index 00000000..651af9f4 --- /dev/null +++ b/scripts/site_imports/journal.py @@ -0,0 +1,267 @@ +"""Shared, import-agnostic journal + undo for reversible bulk imports. + +IMPORT-AGNOSTIC. Nothing here knows about WINC, Maximo, machines, or any one +source. An import writer records the primitive mutations it makes - "set column +X on row Y of table T from old to new" (update) or "insert row Y into table T" +(insert) - each tied to a batch id, and a generic undo replays them in reverse. +That is the whole mechanism: importers emit primitives; undo reverses primitives. + +Identity: one value threads approval, write, and undo. The approval packet's +`batchid` becomes the importbatches primary key, becomes the journal foreign key, +becomes the argument the "Undo import" button passes. Nothing to reconcile. + +Contract (extends the approval packet's 6-step writer contract): + open_batch() -> after the packet passes, write the importbatches row. + record_update()/record_insert() -> append a journal entry in the SAME + transaction as the mutation it describes (ledger rule: the + entry and the change it records commit together). + undo_batch() -> reverse-replay this batch's entries, conservatively. + +Undo is deliberately conservative and never destructive beyond what the import +owns: + - update reverts a cell ONLY if it still holds the value we wrote; a value a + human changed since is left as-is and reported as a conflict (never clobbered). + - insert hard-deletes ONLY the row this batch created, and only when nothing + outside the batch still references it; otherwise it is left and reported. + +All access goes through a SQLAlchemy connection executing parameterised text, so +this reuses the same connection an importer already opened and pulls in no ORM +models - keeping it a true platform primitive any importer can share. +""" + +from collections import namedtuple +from datetime import datetime, timezone + +BATCH_APPLIED = 'applied' +BATCH_UNDONE = 'undone' + +OP_UPDATE = 'update' +OP_INSERT = 'insert' + +# Result of an undo pass, for a report and a UI. +UndoResult = namedtuple('UndoResult', ('batchid', 'reverted', 'conflicts')) + +# A single thing undo could not safely reverse. +Conflict = namedtuple('Conflict', ('operation', 'table', 'targetid', + 'column', 'reason')) + + +def _utcnow(): + return datetime.now(timezone.utc).replace(tzinfo=None) + + +def _as_text(value): + """Store every journalled value as text so a pre/post image compares the + same way it will read back, regardless of the column's declared type.""" + return None if value is None else str(value) + + +def open_batch(connection, packet, createdby=None): + """Write the importbatches row for an approved packet. + + Call once, after the packet has been verified against the current plan and + acknowledged. Returns the batchid so the caller stamps it on every journal + entry it then records. Import-agnostic: the packet's own fields carry all + the identity this needs. + """ + from sqlalchemy import text + + connection.execute(text( + 'INSERT INTO importbatches ' + '(batchid, kind, planhash, rowcount, createdat, createdby, status) ' + 'VALUES (:batchid, :kind, :planhash, :rowcount, :createdat, ' + ':createdby, :status)'), + { + 'batchid': packet['batchid'], + 'kind': packet.get('kind'), + 'planhash': packet.get('planhash'), + 'rowcount': packet.get('rowcount', 0), + 'createdat': _utcnow(), + 'createdby': createdby, + 'status': BATCH_APPLIED, + }) + return packet['batchid'] + + +def record_update(connection, batchid, sequence, table, targetid, column, + oldvalue, newvalue): + """Append an 'update' journal entry describing one cell change. + + Record the pre-image (oldvalue) you read BEFORE applying the change and the + post-image (newvalue) you are about to write, in the same transaction as the + UPDATE itself. + """ + _insert_entry(connection, batchid, sequence, OP_UPDATE, table, targetid, + column, oldvalue, newvalue) + + +def record_insert(connection, batchid, sequence, table, targetid): + """Append an 'insert' journal entry for a row this batch created. + + Only ever emitted for a needs-create row the writer itself inserted, so undo + can hard-delete strictly what the import created. + """ + _insert_entry(connection, batchid, sequence, OP_INSERT, table, targetid, + None, None, None) + + +def _insert_entry(connection, batchid, sequence, operation, table, targetid, + column, oldvalue, newvalue): + from sqlalchemy import text + + connection.execute(text( + 'INSERT INTO importjournalentries ' + '(batchid, sequence, operation, targettable, targetid, columnname, ' + 'oldvalue, newvalue, isreverted) ' + 'VALUES (:batchid, :sequence, :operation, :targettable, :targetid, ' + ':columnname, :oldvalue, :newvalue, 0)'), + { + 'batchid': batchid, + 'sequence': sequence, + 'operation': operation, + 'targettable': table, + 'targetid': targetid, + 'columnname': column, + 'oldvalue': _as_text(oldvalue), + 'newvalue': _as_text(newvalue), + }) + + +def undo_batch(connection, batchid, references=None): + """Reverse-replay an applied batch's journal. Returns an UndoResult. + + references (optional): { table -> [(reftable, refcolumn), ...] } describing + which columns in other tables point at a created row's id. Used to decide + whether an inserted row is safe to hard-delete: if any reference OUTSIDE + this batch's own updates still points at it, the row is left and reported as + a conflict rather than deleted. Omit it to require that the batch's own + update-reverts have already cleared every reference (the common case, since a + created businessunit is only ever pointed at by the same batch's asset + updates). + + Idempotent: a batch already 'undone' is a no-op. Entries are walked in + reverse sequence so inserts are removed only after the updates that pointed + at them are reverted. + """ + from sqlalchemy import text + + batch = connection.execute(text( + 'SELECT status FROM importbatches WHERE batchid = :batchid'), + {'batchid': batchid}).fetchone() + if batch is None: + raise ValueError('unknown batchid: %s' % batchid) + if batch[0] == BATCH_UNDONE: + return UndoResult(batchid, 0, []) + + entries = connection.execute(text( + 'SELECT entryid, sequence, operation, targettable, targetid, ' + 'columnname, oldvalue, newvalue FROM importjournalentries ' + 'WHERE batchid = :batchid AND isreverted = 0 ' + 'ORDER BY sequence DESC, entryid DESC'), + {'batchid': batchid}).fetchall() + + reverted = 0 + conflicts = [] + for row in entries: + (entryid, _sequence, operation, table, targetid, column, + oldvalue, newvalue) = row + + if operation == OP_UPDATE: + conflict = _revert_update(connection, table, targetid, column, + oldvalue, newvalue) + elif operation == OP_INSERT: + conflict = _revert_insert(connection, batchid, table, targetid, + references) + else: + conflict = Conflict(operation, table, targetid, column, + 'unknown operation') + + if conflict is None: + connection.execute(text( + 'UPDATE importjournalentries SET isreverted = 1 ' + 'WHERE entryid = :entryid'), {'entryid': entryid}) + reverted += 1 + else: + conflicts.append(conflict) + + connection.execute(text( + 'UPDATE importbatches SET status = :status, undoneat = :undoneat ' + 'WHERE batchid = :batchid'), + {'status': BATCH_UNDONE, 'undoneat': _utcnow(), 'batchid': batchid}) + + return UndoResult(batchid, reverted, conflicts) + + +def _revert_update(connection, table, targetid, column, oldvalue, newvalue): + """Revert one cell to its pre-image, but only if it still holds what we + wrote. Returns a Conflict if a human changed it since, else None.""" + from sqlalchemy import text + + pk = _primary_key(connection, table) + current = connection.execute(text( + 'SELECT %s FROM %s WHERE %s = :targetid' % (column, table, pk)), + {'targetid': targetid}).fetchone() + if current is None: + return Conflict(OP_UPDATE, table, targetid, column, 'row missing') + + if _as_text(current[0]) != _as_text(newvalue): + return Conflict(OP_UPDATE, table, targetid, column, + 'value changed since import') + + connection.execute(text( + 'UPDATE %s SET %s = :oldvalue WHERE %s = :targetid' + % (table, column, pk)), + {'oldvalue': oldvalue, 'targetid': targetid}) + return None + + +def _revert_insert(connection, batchid, table, targetid, references): + """Hard-delete a row this batch created, only if nothing outside the batch + still references it. Returns a Conflict when it is not safe, else None.""" + from sqlalchemy import text + + pk = _primary_key(connection, table) + for reftable, refcolumn in (references or {}).get(table, []): + referencing = connection.execute(text( + 'SELECT COUNT(*) FROM %s WHERE %s = :targetid' + % (reftable, refcolumn)), {'targetid': targetid}).fetchone()[0] + if referencing: + return Conflict(OP_INSERT, table, targetid, None, + 'still referenced by %s.%s' % (reftable, refcolumn)) + + connection.execute(text( + 'DELETE FROM %s WHERE %s = :targetid' % (table, pk)), + {'targetid': targetid}) + return None + + +def _primary_key(connection, table): + """Single-column primary key name for a table, via the live schema.""" + from sqlalchemy import inspect + + pk = inspect(connection).get_pk_constraint(table) + columns = pk.get('constrained_columns') or [] + if len(columns) != 1: + raise ValueError( + 'journal undo needs a single-column primary key on %s' % table) + return columns[0] + + +def format_undo_result(result): + """Human-readable rendering of an UndoResult for the console or a UI panel.""" + lines = [] + lines.append('UNDO batch %s' % result.batchid) + lines.append(' reverted : %d entries' % result.reverted) + if result.conflicts: + lines.append(' conflicts: %d (left as-is, not reverted)' + % len(result.conflicts)) + for conflict in result.conflicts: + target = conflict.table + if conflict.column: + target = '%s.%s' % (conflict.table, conflict.column) + lines.append(' ! %-8s %s #%s - %s' % ( + conflict.operation, target, conflict.targetid, + conflict.reason)) + else: + lines.append(' conflicts: none') + return '\n'.join(lines) diff --git a/scripts/site_imports/resolver.py b/scripts/site_imports/resolver.py new file mode 100644 index 00000000..e5292ad9 --- /dev/null +++ b/scripts/site_imports/resolver.py @@ -0,0 +1,134 @@ +"""Shared four-state resolver vocabulary for site-import dry runs. + +IMPORT-AGNOSTIC. Nothing here knows about machines, eq1, locations, or any one +site. Every site importer maps a raw exported value onto a database row, and +every one of those mappings lands in exactly one of four states: + + RESOLVED the value maps to a row that already exists; safe to set. + BLANK there is nothing to set (empty cell, or a deliberately + unmappable value); leave the target field as-is. + NEEDS_CREATE the value maps to a valid canonical name, but that row does + not exist yet; a writer may create it, WITH acknowledgment. + UNRESOLVED the value has no mapping at all; a human must add one before + any writer runs. This is the state that blocks approval. + +Keeping this vocabulary in one place is what lets a promotion/approval helper, +an undo journal, and any number of importers all speak the same language. Add +new importers by writing their lookups and alias maps; reuse these primitives to +turn a raw value into a Resolution. +""" + +from collections import OrderedDict, namedtuple + +# The four terminal states a single field resolution can end in. +RESOLVED = 'resolved' +BLANK = 'blank' +NEEDS_CREATE = 'needs-create' +UNRESOLVED = 'unresolved' + +# status: one of the four above. +# id: the database id when RESOLVED, else None. +# name: the canonical target name when known (RESOLVED / NEEDS_CREATE), else +# None. Carried so callers can report and, later, create by name. +Resolution = namedtuple('Resolution', ('status', 'id', 'name')) + + +def norm(value): + """Lowercase, stripped form used for every name comparison and alias key. + + Centralised so a lookup table, an alias map, and the value being resolved + are always normalised the same way. If this ever changes, it changes for + everyone at once, which is the point. + """ + return (value or '').strip().lower() + + +def load_lookup(connection, table, id_column, name_column, display=None): + """norm(name) -> id for every row of any table. One loader, every importer. + + This is the "match against what is actually in the database" primitive. It + reads the live table at call time, so a value a human adds in Settings is + visible on the next run with no code change: a name that exists resolves, + one that does not is NEEDS_CREATE or UNRESOLVED. Every site importer maps its + exported values onto some reference table (locations, businessunits, vendors, + ...) and they should all load that table the same way rather than each + hand-rolling a SELECT. + + Pass a dict as `display` to also collect norm(name) -> original-cased name, + so a report can echo the name the way the database spells it rather than the + normalised key. + + First writer wins on a duplicate normalised name, which is deterministic and + matches how the importers resolve: the ambiguity is a data problem to report, + not something the loader should silently reorder. + """ + from sqlalchemy import text + + out = {} + rows = connection.execute(text( + 'SELECT %s, %s FROM %s' % (id_column, name_column, table))) + for ident, name in rows: + key = norm(name) + if key: + out.setdefault(key, ident) + if display is not None: + display.setdefault(key, (name or '').strip()) + return out + + +def resolve_name(name, lookup_by_name, missing=UNRESOLVED): + """Resolve a value by direct name match against a lookup table. + + lookup_by_name maps norm(name) -> database id. A blank name is BLANK. A hit + is RESOLVED with the id. A miss is `missing` - callers pass UNRESOLVED when a + human must supply a mapping, or NEEDS_CREATE when the name is trusted and a + writer may create it. + """ + if not (name or '').strip(): + return Resolution(BLANK, None, None) + ident = lookup_by_name.get(norm(name)) + if ident is not None: + return Resolution(RESOLVED, ident, name) + return Resolution(missing, None, name) + + +def resolve_alias(raw, alias_map, lookup_by_name, missing=UNRESOLVED): + """Resolve a dirty raw value through an alias map, then a lookup table. + + The alias-first path: a raw export value (possibly truncated or oddly cased) + is normalised and looked up in alias_map to get a canonical name, which is + then resolved against lookup_by_name. + + raw is blank -> BLANK (nothing to set) + raw not in alias_map -> missing (default UNRESOLVED: add an alias) + alias maps to None -> BLANK (deliberately unmappable value) + canonical exists in lookup -> RESOLVED + canonical not yet in lookup -> NEEDS_CREATE + """ + if not (raw or '').strip(): + return Resolution(BLANK, None, None) + key = norm(raw) + if key not in alias_map: + return Resolution(missing, None, None) + canonical = alias_map[key] + if canonical is None: + return Resolution(BLANK, None, None) + return resolve_name(canonical, lookup_by_name, missing=NEEDS_CREATE) + + +def tally(plan, status_column, value_column, status): + """Count rows in a plan whose status_column equals `status`, by value. + + Returns an OrderedDict of value -> count, highest count first. Shared by the + dry-run report and the approval packet so the numbers a human reads and the + numbers a writer enforces come from one place. + """ + counts = {} + for row in plan: + if row.get(status_column) == status: + value = row.get(value_column) or '' + counts[value] = counts.get(value, 0) + 1 + ordered = OrderedDict() + for value, count in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])): + ordered[value] = count + return ordered diff --git a/scripts/site_imports/suggest.py b/scripts/site_imports/suggest.py new file mode 100644 index 00000000..a3ae3f28 --- /dev/null +++ b/scripts/site_imports/suggest.py @@ -0,0 +1,159 @@ +"""Conservative, explainable match suggestions for UNRESOLVED values. + +IMPORT-AGNOSTIC and REPORT-ONLY. This never resolves anything and never writes. +Given a raw value that the normal rules could not map, it proposes zero or more +candidate names that already exist in a lookup table, each with a reason and a +confidence, so a human can accept one into that importer's alias map. Nothing +here is ever applied automatically - a silent wrong match is the worst failure +mode in dirty asset data, so every suggestion has to be acknowledged. + +Two tiers, tried in order, stopping at the first that yields anything: + + 1. Bounded deterministic transforms (confidence 'high'). These are rules that + fit the shapes actually seen in the data - a bay code with a slash is two + bays (T6/U6 -> T6, U6), a parenthetical is a note (LD9W(GE90) -> LD9W), a + decimal is a sub-bay (M6.3 -> M6), a trailing range is its first member + (L1-4 -> L1). Each transform is a named rule, not a fuzzer, so the reason + printed next to a suggestion tells a human exactly why it was proposed. + + 2. A single edit-distance-1 fallback (confidence 'low'). Only if no transform + matched, and only for a one-character difference against an existing name. + Bounded to distance 1 on purpose: it catches a typo or a dropped letter and + nothing looser, so it cannot quietly invent a plausible-looking wrong bay. + +The caller supplies the same normalisation and the lookup it resolves against, so +suggestions are always checked against what is really in the database. +""" + +import re +from collections import namedtuple + +from scripts.site_imports.resolver import norm + +# candidate: the existing name we think the raw value meant. +# reason: which rule proposed it (shown to the human). +# confidence: 'high' for a deterministic transform, 'low' for edit-distance. +Suggestion = namedtuple('Suggestion', ('candidate', 'reason', 'confidence')) + +_PAREN = re.compile(r'\s*\([^)]*\)') + + +def _candidates_from_transforms(stripped): + """Yield (transformed_name, reason) from the bounded deterministic rules. + + Operates on the value AFTER the importer has removed its own prefix (e.g. + WM-). Each rule yields plain names; the caller checks each against the live + lookup, so a rule that produces a non-existent name simply contributes no + suggestion. + """ + # A parenthetical note: LD9W(GE90) -> LD9W + if '(' in stripped: + yield (_PAREN.sub('', stripped).strip(), 'dropped parenthetical note') + + # A slash means two bays: T6/U6 -> T6 and U6. A bare second half (9/10) + # inherits the first half's letter prefix (T9/10 -> T9, T10). + if '/' in stripped: + left, right = (part.strip() for part in stripped.split('/', 1)) + if left: + yield (left, 'first of a slash-joined pair') + if right: + yield (right, 'second of a slash-joined pair') + prefix = re.match(r'^([A-Za-z]+)', left) + if prefix and re.match(r'^\d', right): + yield (prefix.group(1) + right, + 'second of a slash-joined pair, prefix carried') + + # A decimal sub-bay: M6.3 -> M6 + if '.' in stripped: + yield (stripped.split('.', 1)[0].strip(), 'dropped decimal sub-bay') + + # A trailing numeric range: L1-4 -> L1 + dash = re.match(r'^([A-Za-z]*\d+)-\d+$', stripped) + if dash: + yield (dash.group(1), 'first of a numeric range') + + +def _edit_distance_at_most_one(left, right): + """True if `left` and `right` differ by at most one insert/delete/replace. + + Bounded on purpose. A length gap over one is an immediate no; otherwise walk + both once and allow a single mismatch. Cheap and strict - it is a typo + catcher, not a similarity search. + """ + if left == right: + return True + length_left, length_right = len(left), len(right) + if abs(length_left - length_right) > 1: + return False + if length_left == length_right: + return sum(a != b for a, b in zip(left, right)) == 1 + # Ensure left is the shorter one, then try to match with one insertion. + if length_left > length_right: + left, right = right, left + length_left, length_right = length_right, length_left + index_left = index_right = 0 + skipped = False + while index_left < length_left and index_right < length_right: + if left[index_left] == right[index_right]: + index_left += 1 + index_right += 1 + elif skipped: + return False + else: + skipped = True + index_right += 1 + return True + + +# A low-confidence edit-distance pass can match many short bay codes at once, +# which is noise, not help. Cap how many it emits so a human sees the few most +# plausible typo-fixes and nothing drowns the high-confidence transforms. +_MAX_LOW_CONFIDENCE = 3 + + +def suggest(raw_value, stripped_value, lookup_by_name, display=None): + """Suggestions for one unresolved value. Empty list means "no idea". + + raw_value the original export string (for reporting only). + stripped_value the value after the importer removed its own prefix; this is + what the transforms and the edit-distance search operate on. + lookup_by_name norm(name) -> id, the live table to match against. + display optional norm(name) -> original-cased name, so a suggestion + echoes the database's spelling instead of the normalised key. + + Deterministic transforms first; only if none of them hits an existing name + does the edit-distance-1 fallback run, and that fallback is capped. High- + confidence suggestions are never diluted by low-confidence ones. + """ + display = display or {} + + def spell(key): + return display.get(key, key) + + results = [] + seen = set() + for candidate, reason in _candidates_from_transforms(stripped_value): + candidate = (candidate or '').strip() + if not candidate: + continue + key = norm(candidate) + if key in lookup_by_name and key not in seen: + seen.add(key) + results.append(Suggestion(spell(key), reason, 'high')) + if results: + return results + + target = norm(stripped_value) + if not target: + return results + low = [] + for existing_key in lookup_by_name: + if existing_key not in seen and _edit_distance_at_most_one( + target, existing_key): + seen.add(existing_key) + low.append(Suggestion(spell(existing_key), + 'one-character difference', 'low')) + # Only surface the capped set; too many one-off matches is noise. + if len(low) <= _MAX_LOW_CONFIDENCE: + results.extend(sorted(low)) + return results diff --git a/scripts/site_imports/winc/__init__.py b/scripts/site_imports/winc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/site_imports/winc/aliases.py b/scripts/site_imports/winc/aliases.py new file mode 100644 index 00000000..2c847fbd --- /dev/null +++ b/scripts/site_imports/winc/aliases.py @@ -0,0 +1,67 @@ +"""Alias and normalization tables for the WINC machine adoption dry-run. + +This file is DATA, not logic. It is the single place a human edits when the +analyzer reports an unresolved value. Keep it boring: plain dicts, one entry per +line, lowercase keys. The analyzer lowercases and strips before it looks a value +up here, so keys here must be lowercase and already stripped. + +Two independent problems get solved here: + +1. eq1 -> business unit. The May export's eq1 column is a free-text value-stream + label (LG PARTS, SM SPOOLS, Large Spoo, HPTCR ...). It is dirty: truncated at + ten characters, inconsistently cased, and it does NOT match anything in the + live businessunits table (that table is a list of offices auto-created by an + earlier printer import). So every eq1 value must be mapped by hand to the + canonical business-unit name we want on the asset. A canonical name that does + not yet exist in the database is reported by the analyzer as "would need to + be created" - the analyzer never invents one and never writes one. + +2. location code overrides. Most WM- codes resolve by simply dropping the + WM- prefix and matching a location by name (WM-J3 -> J3). A handful do not, + because the bay code carries a slash, a note in parentheses, or a room name + the location table spells differently. Those exceptions - and only those - + live here, mapping the RAW export location string to the exact locationname + already in the database. +""" + +# eq1 (lowercased, stripped) -> canonical business unit name. +# The value on the right is what we intend to store; the analyzer checks whether +# it already exists in businessunits and flags the ones that do not. +EQ1_TO_BUSINESSUNIT = { + 'lg parts': 'Large Parts', + 'lg part': 'Large Parts', + 'large part': 'Large Parts', + 'sm spools': 'Small Spools', + 'sm spool': 'Small Spools', + 'large spoo': 'Large Spools', + 'lg spools': 'Large Spools', + 'lg spool': 'Large Spools', + 'hptcr': 'HPT Rotor', + 'bliskcr': 'Blisk Rotor', + 'blisk': 'Blisk Rotor', + 'leap x': 'LEAP-X', +} + +# Raw export location string (lowercased, stripped) -> exact locationname in DB. +# Only for codes that do NOT resolve by dropping the WM- prefix. Leave a value +# as None to say "known to be unmappable, do not touch location for this row". +# +# The entries below the two None rows were promoted from the dry-run's +# high-confidence suggestions and verified to exist in the locations table. A +# slash-joined bay code (T6/U6) names two adjacent bays, but an asset carries a +# single locationid, so each maps to the FIRST (primary) bay - the deterministic +# choice a human confirmed. LD7W and LD6W are intentionally absent: they had no +# existing match and need a location created before they can be aliased. +LOCATION_OVERRIDES = { + 'wm': None, + 'wm-toolroom': None, + 'wm-t6/u6': 'T6', + 'wm-t9/10': 'T9', + 'wm-u9/10': 'U9', + 'wm-t4/u4': 'T4', + 'wm-ld9w(ge90)': 'LD9W', + 'wm-m6.3': 'M6', + 'wm-l1-4': 'L1', + 'wm-ld7w': None, + 'wm-ld6w': None, +} diff --git a/scripts/site_imports/winc/dry_run.py b/scripts/site_imports/winc/dry_run.py new file mode 100644 index 00000000..fc14a1b3 --- /dev/null +++ b/scripts/site_imports/winc/dry_run.py @@ -0,0 +1,385 @@ +"""Dry-run analyzer for the WINC (West Manufacturing) machine adoption. + +READ ONLY. This script opens the database, reads the May asset export, decides +what an import WOULD change, and prints a report. It never writes. Nothing here +commits, updates, or inserts - that is a separate tool, written only once this +report reads clean and a human has signed off on it. + +What it does, in order: + + 1. Load every existing machine's asset number and build the crosswalk key + (the workstation number, e.g. 0083A) the export will be matched on. + 2. Read the export with a real CSV parser (quoted commas, BOM) and, for each + row that carries a workstation number, decide: + - does it match an existing machine? (in scope vs skipped) + - what name would we set? (export description) + - what location would we set? (WM- -> locations, via + aliases.LOCATION_OVERRIDES for + the exceptions) + - what business unit would we set? (eq1 -> aliases.EQ1_TO_ + BUSINESSUNIT -> businessunits) + 3. Classify each field resolution as RESOLVED, BLANK (nothing to set, leave + as-is), NEEDS-CREATE (target value is valid but not yet in the database), + or UNRESOLVED (no alias entry - a human must add one to aliases.py). + 4. Print a summary and, with --rows, a per-row table. With --csv PATH it also + writes the full plan so the numbers can be diffed between runs. + +Run it against the real database read-only: + + python -m scripts.site_imports.winc.dry_run --export "exports/ASSET_current May26.csv" + python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --rows + python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --csv plan.csv + +The design goal is reliability over cleverness: one match rule, one location +rule, one business-unit rule, every exception in aliases.py, and a report that +tells a human exactly what is unresolved so the alias table can be completed +before anyone runs the writer. +""" + +import argparse +import csv +import os +import re +import sys +from collections import OrderedDict +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv + +# Same reasoning as scripts/export_catalog.py: a plain `python -m` run gets +# neither wsgi.py's nor the flask CLI's .env load, so DATABASE_URL would be +# unset and config would silently fall back to a hardcoded localhost URL. Load +# it here before anything reads config. +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from scripts.site_imports.approval import build_packet, format_packet +from scripts.site_imports.suggest import suggest +from scripts.site_imports.resolver import ( + BLANK, + NEEDS_CREATE, + RESOLVED, + Resolution, + UNRESOLVED, + load_lookup, + norm as _norm, + resolve_alias, + resolve_name, +) +from scripts.site_imports.winc.aliases import ( + EQ1_TO_BUSINESSUNIT, + LOCATION_OVERRIDES, +) + +# WM- is the only bay prefix in the WINC export; a location code resolves by +# dropping it and matching a location by name. Everything else is an override. +_WM_PREFIX = re.compile(r'^WM-?', re.IGNORECASE) + +# A workstation number is a LEADING run of digits with an optional single letter +# suffix. The export writes it clean (W0083A); the machines table stores the +# same number but often with a trailing descriptor (0083 Debur, 0083A Inspect). +# So the key is the leading token on both sides, matched unanchored - anchoring +# with $ would drop every existing machine whose name carries a descriptor and +# silently shrink the in-scope set. The leading W is the single biggest gotcha +# in this data, so it is stripped here in exactly one place. +_WSNUM = re.compile(r'^(\d{3,5}[A-Z]?)') + + +def workstation_key(assetnum): + """The crosswalk key (e.g. '0083A') for an asset number, or None. + + Strips a leading W and keeps the leading run of digits plus an optional + single trailing letter, ignoring any descriptor that follows. Returns None + for anything that is not a workstation number (blank rows, facilities codes, + named test machines) so non-machine rows fall out of scope on their own + rather than being force-fit. + """ + text = (assetnum or '').strip().upper() + if text.startswith('W'): + text = text[1:] + match = _WSNUM.match(text) + return match.group(1) if match else None + + +def load_existing_machine_keys(connection): + """Map workstation key -> list of existing machine asset numbers. + + A list, not a single value, because the machines table has a few numbers + that collide once reduced to their workstation key (0083 exists as + '0083 Debur' and '0083 Inspection'). The analyzer reports those as ambiguous + rather than guessing which one the export row means. + """ + from sqlalchemy import text + + keys = OrderedDict() + rows = connection.execute(text( + "SELECT a.assetnumber " + "FROM assets a JOIN machines m ON m.assetid = a.assetid")) + for (assetnumber,) in rows: + key = workstation_key(assetnumber) + if key: + keys.setdefault(key, []).append(assetnumber) + return keys + + +def load_locations(connection, display=None): + """locationname (normalized) -> locationid, from the live locations table.""" + return load_lookup(connection, 'locations', 'locationid', 'locationname', + display=display) + + +def load_businessunits(connection): + """businessunit (normalized) -> businessunitid, from the live table.""" + return load_lookup(connection, 'businessunits', 'businessunitid', + 'businessunit') + + +def resolve_location(raw_location, locations_by_name): + """Resolution for a raw WM bay code. + + Rule: an override in aliases wins; otherwise drop the WM- prefix and match a + location by name. A blank export cell means "leave the asset's location as + is" and is reported as BLANK, not UNRESOLVED, because there is nothing to + set and nothing for a human to fix. Overrides are checked first because they + carry the exceptions the plain prefix rule cannot handle. + """ + raw = (raw_location or '').strip() + if not raw: + return Resolution(BLANK, None, None) + + if _norm(raw) in LOCATION_OVERRIDES: + return resolve_alias(raw, LOCATION_OVERRIDES, locations_by_name) + + name = _WM_PREFIX.sub('', raw).strip() + return resolve_name(name, locations_by_name) + + +def resolve_businessunit(raw_eq1, businessunits_by_name): + """Resolution for an eq1 value stream. + + A blank eq1 is BLANK (leave the asset alone). A non-blank value must have an + alias entry; without one it is UNRESOLVED so a human adds it to aliases.py. A + mapped canonical name that does not yet exist in businessunits is + NEEDS-CREATE - flagged, never created here. + """ + return resolve_alias(raw_eq1, EQ1_TO_BUSINESSUNIT, businessunits_by_name) + + +def read_export(path): + """Rows of the export as dicts. utf-8-sig handles the BOM; the csv module + handles the quoted commas embedded in some descriptions.""" + with open(path, newline='', encoding='utf-8-sig') as handle: + return list(csv.DictReader(handle)) + + +def parse_export_text(text): + """Rows of an export given its raw CSV text (same shape as read_export). + + Split out so an API route can hand the analyzer an uploaded file's bytes + without writing them to disk, while the CLI keeps reading from a path. + """ + import io + + return list(csv.DictReader(io.StringIO(text))) + + +def analyze(connection, export_path): + """Build the per-row plan from a file path. Returns (plan, summary, lookups).""" + return analyze_rows(connection, read_export(export_path)) + + +def analyze_rows(connection, export_rows): + """Build the per-row plan. Returns (plan_rows, summary_dict, lookups). + + Takes already-parsed export rows so the same analysis serves both the CLI + (which reads a path) and the API route (which reads uploaded bytes). + """ + existing = load_existing_machine_keys(connection) + location_names = {} + locations = load_locations(connection, display=location_names) + businessunits = load_businessunits(connection) + + plan = [] + summary = OrderedDict([ + ('export_rows', len(export_rows)), + ('non_workstation_rows', 0), + ('in_scope', 0), + ('skipped_no_existing_machine', 0), + ('ambiguous_match', 0), + ('location_' + RESOLVED, 0), + ('location_' + BLANK, 0), + ('location_' + NEEDS_CREATE, 0), + ('location_' + UNRESOLVED, 0), + ('businessunit_' + RESOLVED, 0), + ('businessunit_' + BLANK, 0), + ('businessunit_' + NEEDS_CREATE, 0), + ('businessunit_' + UNRESOLVED, 0), + ]) + + for row in export_rows: + assetnum = (row.get('assetnum') or '').strip() + key = workstation_key(assetnum) + if key is None: + summary['non_workstation_rows'] += 1 + continue + + matches = existing.get(key, []) + if not matches: + summary['skipped_no_existing_machine'] += 1 + continue + + summary['in_scope'] += 1 + ambiguous = len(matches) > 1 + if ambiguous: + summary['ambiguous_match'] += 1 + + loc = resolve_location(row.get('location'), locations) + bu = resolve_businessunit(row.get('eq1'), businessunits) + summary['location_' + loc.status] += 1 + summary['businessunit_' + bu.status] += 1 + + plan.append(OrderedDict([ + ('assetnum', assetnum), + ('workstation_key', key), + ('matched_asset', '|'.join(matches)), + ('ambiguous', 'yes' if ambiguous else ''), + ('name', (row.get('description') or '').strip()), + ('raw_location', (row.get('location') or '').strip()), + ('location_status', loc.status), + ('locationid', loc.id if loc.id is not None else ''), + ('location_name', loc.name or ''), + ('raw_eq1', (row.get('eq1') or '').strip()), + ('businessunit_status', bu.status), + ('businessunitid', bu.id if bu.id is not None else ''), + ('businessunit_name', bu.name or ''), + ])) + + return plan, summary, {'locations': locations, + 'location_names': location_names, + 'businessunits': businessunits} + + +def print_summary(summary): + print('=' * 60) + print('WINC machine adoption - DRY RUN (no changes written)') + print('=' * 60) + for label, value in summary.items(): + print(' %-32s %6d' % (label.replace('_', ' '), value)) + + +def print_unresolved(plan, lookups=None): + loc = OrderedDict() + bu = OrderedDict() + for row in plan: + if row['location_status'] == UNRESOLVED: + loc[row['raw_location']] = loc.get(row['raw_location'], 0) + 1 + if row['businessunit_status'] == UNRESOLVED: + bu[row['raw_eq1']] = bu.get(row['raw_eq1'], 0) + 1 + needs_loc = OrderedDict() + needs_bu = OrderedDict() + for row in plan: + if row['location_status'] == NEEDS_CREATE: + needs_loc[row['location_name']] = needs_loc.get(row['location_name'], 0) + 1 + if row['businessunit_status'] == NEEDS_CREATE: + needs_bu[row['businessunit_name']] = needs_bu.get(row['businessunit_name'], 0) + 1 + + if loc: + print('\nUNRESOLVED locations (add to aliases.LOCATION_OVERRIDES):') + location_lookup = (lookups or {}).get('locations') or {} + location_names = (lookups or {}).get('location_names') or {} + for value, count in sorted(loc.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + stripped = _WM_PREFIX.sub('', value).strip() + for hint in suggest(value, stripped, location_lookup, + display=location_names): + print(' -> suggest "%s" (%s, confidence %s)' + % (hint.candidate, hint.reason, hint.confidence)) + if bu: + print('\nUNRESOLVED eq1 (add to aliases.EQ1_TO_BUSINESSUNIT):') + for value, count in sorted(bu.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + if needs_loc: + print('\nNEEDS-CREATE locations (mapped, but not yet in locations):') + for value, count in sorted(needs_loc.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + if needs_bu: + print('\nNEEDS-CREATE business units (mapped, but not yet in businessunits):') + for value, count in sorted(needs_bu.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + + +def print_rows(plan): + print('\nPER-ROW PLAN:') + header = ('assetnum', 'matched_asset', 'name', 'raw_location', + 'location_status', 'raw_eq1', 'businessunit_status') + print(' ' + ' | '.join(header)) + for row in plan: + print(' ' + ' | '.join(str(row[h]) for h in header)) + + +def write_csv(plan, path): + if not plan: + return + with open(path, 'w', newline='', encoding='utf-8') as handle: + writer = csv.DictWriter(handle, fieldnames=list(plan[0].keys())) + writer.writeheader() + writer.writerows(plan) + + +def build_app_connection(): + """Yield a live SQLAlchemy connection inside an app context. + + Returns (app, connection). Caller is responsible for closing nothing - the + connection lives for the app context and the process exits right after. + """ + from shopdb import create_app + from shopdb.api import db + + app = create_app() + context = app.app_context() + context.push() + return app, db.session.connection() + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Dry-run analyzer for WINC machine adoption. Read only.') + parser.add_argument('--export', required=True, + help='Path to the ASSET export CSV.') + parser.add_argument('--rows', action='store_true', + help='Print the full per-row plan.') + parser.add_argument('--csv', dest='csv_out', + help='Write the full per-row plan to this CSV path.') + parser.add_argument('--packet', action='store_true', + help='Print the approval packet (hash, needs-create ' + 'and blocking summary) a writer would enforce.') + args = parser.parse_args(argv) + + if not os.path.exists(args.export): + parser.error('export file not found: %s' % args.export) + + _app, connection = build_app_connection() + plan, summary, lookups = analyze(connection, args.export) + + print_summary(summary) + print_unresolved(plan, lookups) + if args.rows: + print_rows(plan) + if args.csv_out: + write_csv(plan, args.csv_out) + print('\nWrote per-row plan: %s (%d rows)' % (args.csv_out, len(plan))) + if args.packet: + packet = build_packet( + 'winc-machine-adoption', plan, summary, + [('businessunit', 'businessunit_status', 'raw_eq1', + 'businessunit_name'), + ('location', 'location_status', 'raw_location', 'location_name')]) + print('\n' + format_packet(packet)) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/site_imports/winc/undo.py b/scripts/site_imports/winc/undo.py new file mode 100644 index 00000000..7adec6a8 --- /dev/null +++ b/scripts/site_imports/winc/undo.py @@ -0,0 +1,55 @@ +"""WINC machine adoption - UNDO (reverse an applied import batch). + +A thin CLI over the shared, import-agnostic journal.undo_batch. It is WINC-named +only for discoverability; the actual work is generic, so the same reversal serves +any importer that wrote through the journal. Reverting is conservative: a cell a +human changed since the import is left as-is and reported, and a created business +unit is deleted only when nothing outside the batch still references it. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv +from pathlib import Path + +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from scripts.site_imports import journal +from scripts.site_imports.winc.dry_run import build_app_connection +from scripts.site_imports.winc.writer import _UNDO_REFERENCES + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Undo an applied WINC import batch (reversible journal).') + parser.add_argument('--batch', required=True, + help='The batch id to reverse.') + parser.add_argument('--commit', action='store_true', + help='Actually reverse. Without it, this is a no-op.') + args = parser.parse_args(argv) + + if not args.commit: + print('DRY: pass --commit to reverse batch %s' % args.batch) + return 0 + + _app, connection = build_app_connection() + from shopdb.api import db + try: + result = journal.undo_batch(connection, args.batch, + references=_UNDO_REFERENCES) + db.session.commit() + except Exception: + db.session.rollback() + raise + + print(journal.format_undo_result(result)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/site_imports/winc/writer.py b/scripts/site_imports/winc/writer.py new file mode 100644 index 00000000..ce18d3b9 --- /dev/null +++ b/scripts/site_imports/winc/writer.py @@ -0,0 +1,278 @@ +"""WINC machine adoption - WRITER (applies the approved plan, reversibly). + +This is the write half of the WINC import; scripts/site_imports/winc/dry_run.py +is the read-only half. The writer never invents its own view of the data: it +rebuilds the exact same plan the dry-run produced, verifies that plan against the +approval packet the human signed off on, and only then writes - recording every +change to the shared import journal so the whole batch can be undone. + +The contract, in order (the one documented in approval.py, made concrete here): + + 1. rebuild the plan from the same export + live DB, + 2. build a packet from it, + 3. compare planhash to the approved packet - abort on mismatch (drift), + 4. refuse if the packet is blocking (any unresolved value), + 5. require explicit acknowledgment for every needs-create group, + 6. write inside ONE transaction, stamping the batch id onto the journal: + a. create each acknowledged needs-create business unit (journal insert), + b. update each in-scope asset's name / location / business unit + (journal update, capturing the pre-image first). + +Everything commits together or not at all. Ambiguous rows (one workstation key +matching several machines) are never written - the dry-run flags them and the +writer skips them, because guessing which machine an export row means is exactly +the kind of cleverness this import refuses. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv +from pathlib import Path + +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from scripts.site_imports import journal +from scripts.site_imports.approval import build_packet, packet_matches_plan +from scripts.site_imports.resolver import NEEDS_CREATE, RESOLVED, norm +from scripts.site_imports.winc.dry_run import ( + analyze, + build_app_connection, +) + +# The packet groups the WINC dry-run declares, reused verbatim so the writer +# hashes and reads the plan exactly the way approval did. +_PACKET_GROUPS = [ + ('businessunit', 'businessunit_status', 'raw_eq1', 'businessunit_name'), + ('location', 'location_status', 'raw_location', 'location_name'), +] + +# References into a created row's id, so undo can tell whether an inserted +# business unit is still pointed at by something outside this batch before it +# hard-deletes it. Shared with the undo CLI and the web undo path. +_UNDO_REFERENCES = { + 'businessunits': [('assets', 'businessunitid')], +} + + +# References into a created row's id, so undo can tell whether an inserted +# business unit is still pointed at by something outside this batch before it +# hard-deletes it. +# Max width of assets.name; the export description can exceed it (a few rows +# carry a whole safety notice in the field), so a set name is clamped rather +# than letting the whole transaction fail on one over-long row. The pre-image is +# journalled, so the clamp is reversible like any other change. +_ASSET_NAME_MAXLEN = 100 + + +class WriterError(Exception): + """The plan cannot be written as approved (drift, blocking, or missing + acknowledgment). Carries a human-readable reason; never a partial write.""" + + +def _load_asset_ids(connection): + """assetnumber -> assetid for every asset, so a plan row (which carries the + matched asset *number*) can be turned into the id the update targets.""" + from sqlalchemy import text + + out = {} + for assetid, assetnumber in connection.execute(text( + 'SELECT assetid, assetnumber FROM assets')): + out[assetnumber] = assetid + return out + + +def _create_businessunit(connection, batchid, sequence, name): + """Insert one business unit and journal it. Returns (new_id, next_sequence). + + Only ever called for an acknowledged needs-create name, so the journal's + insert entry describes strictly a row this batch created.""" + from sqlalchemy import text + + now = journal._utcnow() + result = connection.execute(text( + 'INSERT INTO businessunits ' + '(businessunit, createddate, modifieddate, isactive) ' + 'VALUES (:name, :now, :now, 1)'), + {'name': name, 'now': now}) + new_id = result.lastrowid + journal.record_insert(connection, batchid, sequence, 'businessunits', + new_id) + return new_id, sequence + 1 + + +def _update_cell(connection, batchid, sequence, table, targetid, column, + oldvalue, newvalue): + """Apply one cell update and journal its pre/post image. Returns the next + sequence. Skips (and does not journal) a no-op where the value is unchanged.""" + from sqlalchemy import text + + if journal._as_text(oldvalue) == journal._as_text(newvalue): + return sequence + connection.execute(text( + 'UPDATE %s SET %s = :newvalue WHERE %s = :targetid' + % (table, column, _asset_pk(table))), + {'newvalue': newvalue, 'targetid': targetid}) + journal.record_update(connection, batchid, sequence, table, targetid, + column, oldvalue, newvalue) + return sequence + 1 + + +def _asset_pk(table): + # The writer only updates the assets table; keep the mapping explicit rather + # than reflecting, so an unexpected table name fails loudly. + return {'assets': 'assetid'}[table] + + +def apply_plan(connection, plan, approved_packet, acknowledge=None, + createdby=None): + """Apply an approved WINC plan, reversibly. Returns the batch id. + + connection a live SQLAlchemy connection in an open transaction. + plan the per-row plan from dry_run.analyze (rebuilt fresh). + approved_packet the packet a human signed off on. + acknowledge set of needs-create group names the human confirmed + (e.g. {'businessunit'}); required for every group the + packet lists under needscreate. + createdby who acknowledged, stamped on the batch. + + Raises WriterError before touching anything if the plan drifted, is + blocking, or a needs-create group was not acknowledged. + """ + acknowledge = set(acknowledge or ()) + + # (2)-(3) rebuild the packet and prove nothing drifted since approval. + current_packet = build_packet('winc-machine-adoption', plan, + approved_packet.get('counts', {}), + _PACKET_GROUPS) + if not packet_matches_plan(approved_packet, plan): + raise WriterError( + 'plan has changed since approval (hash mismatch) - re-run the ' + 'dry-run and re-approve before importing') + + # (4) never write over an unresolved value. + if current_packet.get('blocking'): + raise WriterError( + 'plan is blocking: unresolved values remain - add the missing ' + 'aliases and re-run the dry-run') + + # (5) every needs-create group must be explicitly acknowledged. + missing_ack = set(current_packet.get('needscreate', {})) - acknowledge + if missing_ack: + raise WriterError( + 'needs-create not acknowledged for: %s - confirm these will be ' + 'created before importing' % ', '.join(sorted(missing_ack))) + + batchid = approved_packet['batchid'] + journal.open_batch(connection, approved_packet, createdby=createdby) + sequence = 1 + + # (6a) create acknowledged needs-create business units once each, first, so + # the asset updates below can point at them. + created_bu = {} + for row in plan: + if row['businessunit_status'] != NEEDS_CREATE: + continue + name = row['businessunit_name'] + key = norm(name) + if key not in created_bu: + created_bu[key], sequence = _create_businessunit( + connection, batchid, sequence, name) + + # (6b) update each in-scope, unambiguous asset. + asset_ids = _load_asset_ids(connection) + from sqlalchemy import text + for row in plan: + if row['ambiguous']: + continue + assetnumber = row['matched_asset'] + assetid = asset_ids.get(assetnumber) + if assetid is None: + continue + + current = connection.execute(text( + 'SELECT name, locationid, businessunitid FROM assets ' + 'WHERE assetid = :assetid'), {'assetid': assetid}).fetchone() + cur_name, cur_location, cur_bu = current + + if row['name']: + new_name = row['name'][:_ASSET_NAME_MAXLEN] + sequence = _update_cell(connection, batchid, sequence, 'assets', + assetid, 'name', cur_name, new_name) + + if row['location_status'] == RESOLVED and row['locationid'] != '': + sequence = _update_cell(connection, batchid, sequence, 'assets', + assetid, 'locationid', cur_location, + row['locationid']) + + new_bu = None + if row['businessunit_status'] == RESOLVED and row['businessunitid'] != '': + new_bu = row['businessunitid'] + elif row['businessunit_status'] == NEEDS_CREATE: + new_bu = created_bu[norm(row['businessunit_name'])] + if new_bu is not None: + sequence = _update_cell(connection, batchid, sequence, 'assets', + assetid, 'businessunitid', cur_bu, new_bu) + + return batchid + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='WINC machine adoption writer. Applies an approved plan ' + 'reversibly. Requires --commit to write.') + parser.add_argument('--export', required=True, + help='Path to the ASSET export CSV.') + parser.add_argument('--acknowledge', action='append', default=[], + metavar='GROUP', + help='Acknowledge a needs-create group (repeatable), ' + 'e.g. --acknowledge businessunit.') + parser.add_argument('--createdby', default=None, + help='Who is approving this import (stamped on batch).') + parser.add_argument('--commit', action='store_true', + help='Actually write. Without it, this is a no-op that ' + 'reports what would happen.') + args = parser.parse_args(argv) + + if not os.path.exists(args.export): + parser.error('export file not found: %s' % args.export) + + _app, connection = build_app_connection() + plan, summary, _lookups = analyze(connection, args.export) + packet = build_packet('winc-machine-adoption', plan, summary, + _PACKET_GROUPS) + + if not args.commit: + print('DRY: rebuilt plan of %d rows, hash %s' % ( + packet['rowcount'], packet['planhash'])) + print('DRY: pass --commit with --acknowledge for each needs-create ' + 'group to write. Needs-create groups: %s' + % (', '.join(packet.get('needscreate', {})) or 'none')) + return 0 + + from shopdb.api import db + try: + batchid = apply_plan(connection, plan, packet, + acknowledge=args.acknowledge, + createdby=args.createdby) + db.session.commit() + except WriterError as error: + db.session.rollback() + print('REFUSED: %s' % error) + return 2 + except Exception: + db.session.rollback() + raise + + print('COMMITTED batch %s' % batchid) + print(' to undo: python -m scripts.site_imports.winc.undo --batch %s' + % batchid) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/shopdb/__init__.py b/shopdb/__init__.py index e85f4b67..7073f394 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -104,7 +104,13 @@ from .plugins import plugin_manager # defeated by rotating the header. The helper resolves from remote_addr, which # ProxyFix has already filled from the header when TRUSTED_PROXY_COUNT says a # proxy is really in front. Additive names on the surface, minor bump. -__contract_version__ = '0.29.0' +# 0.30.0: added ImportBatch, ImportJournalEntry, and the BATCH_*/OP_* constants +# to shopdb.api so a plugin's bulk-import UI can list applied import batches and +# reverse one through the shared reversible journal without importing core model +# internals. The journal records the primitive mutations a bulk import makes +# (update a cell, insert a row) tied to a batch id, and a generic undo replays +# them in reverse. Additive names on the surface, minor bump. +__contract_version__ = '0.30.0' # Product release version (see ADR-007). The product version and the # plugin-contract version above are distinct series with independent diff --git a/shopdb/api/__init__.py b/shopdb/api/__init__.py index 19d7626e..db33a9d6 100644 --- a/shopdb/api/__init__.py +++ b/shopdb/api/__init__.py @@ -140,6 +140,22 @@ from shopdb.utils.mailer import send_email, send_alert, send_webhook from shopdb.utils.cmmc_usb_db import cmmc_usb_connection +# Reversible bulk-import journal. A plugin that drives a bulk import (a site +# asset adoption, an integration sync) records the primitive changes it makes so +# the whole batch can be undone. ImportBatch is the identity approval, write, and +# undo all reference; ImportJournalEntry is one reversible mutation. Exposed so a +# plugin's import UI can list applied batches and reverse one without reaching +# into core internals. See docs/PLUGIN-HOOKS.md. +from shopdb.core.models import ( + ImportBatch, + ImportJournalEntry, + BATCH_APPLIED, + BATCH_UNDONE, + OP_UPDATE, + OP_INSERT, +) + + def audit_log( action: str, entitytype: str, diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index 939bf1bd..e550400b 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -595,6 +595,123 @@ def import_asset_csvs(): }) +def _winc_upload_text(): + """Read the single uploaded WINC export as text, or (None, error_response).""" + uploads = request.files.getlist('files') + if not uploads: + return None, error_response(ErrorCodes.VALIDATION_ERROR, + 'No export file was uploaded') + upload = uploads[0] + name = upload.filename or '' + if not name.lower().endswith('.csv'): + return None, error_response(ErrorCodes.VALIDATION_ERROR, + 'The WINC export must be a .csv file') + try: + return upload.read().decode('utf-8-sig'), None + except (UnicodeDecodeError, OSError): + return None, error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + + +@settings_bp.route('/asset-import/winc/preview', methods=['POST']) +@require_role('admin') +def winc_import_preview(): + """Read-only WINC analysis: report what a commit would do. Writes nothing. + + WINC (a Maximo-style asset dump) is not a canonical-CSV source: it updates + existing machines only, needs acknowledgment before creating business units, + and is reversible. This returns the approval packet the UI renders - counts, + the business units it would create, and any unresolved values that block it. + """ + from shopdb.core.services.wincimport import preview, WincImportError + + export_text, failure = _winc_upload_text() + if failure is not None: + return failure + try: + report = preview(export_text) + except WincImportError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + finally: + # A preview must leave nothing behind even though it only read: the + # analyzer runs against the request-scoped session. + db.session.rollback() + return success_response(report) + + +@settings_bp.route('/asset-import/winc/commit', methods=['POST']) +@require_role('admin') +def winc_import_commit(): + """Apply a WINC plan reversibly. Requires acknowledgment for needs-create. + + The 'acknowledge' form field is a comma-separated list of needs-create group + names the admin confirmed (e.g. 'businessunit'). The whole write is one + transaction recorded to the import journal, so it can be undone later. + """ + from shopdb.core.services.wincimport import commit as winc_commit + from shopdb.core.services.wincimport import WincImportError + + export_text, failure = _winc_upload_text() + if failure is not None: + return failure + + acknowledge = [ + part.strip() for part in + (request.form.get('acknowledge') or '').split(',') if part.strip()] + + try: + report = winc_commit(export_text, acknowledge=acknowledge, + createdby=get_jwt_identity()) + AuditLog.log( + 'imported', 'SiteAssets', entityname='winc-machine-adoption', + changes={'batchid': report.get('batchid'), + 'rows': report.get('rowcount')}) + db.session.commit() + except WincImportError as exc: + db.session.rollback() + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + except Exception: + db.session.rollback() + raise + return success_response(report) + + +@settings_bp.route('/asset-import/winc/batches', methods=['GET']) +@require_role('admin') +def winc_import_batches(): + """List applied/undone WINC import batches, newest first (for the undo UI).""" + from shopdb.core.services.wincimport import list_batches + + return success_response({'batches': list_batches()}) + + +@settings_bp.route('/asset-import/winc/undo', methods=['POST']) +@require_role('admin') +def winc_import_undo(): + """Reverse an applied WINC import batch by id. Conservative and reversible.""" + from shopdb.core.services.wincimport import undo as winc_undo + + payload = request.get_json(silent=True) or {} + batchid = (payload.get('batchid') or request.form.get('batchid') or '').strip() + if not batchid: + return error_response(ErrorCodes.VALIDATION_ERROR, 'A batchid is required') + + try: + result = winc_undo(batchid) + AuditLog.log( + 'undone', 'SiteAssets', entityname='winc-machine-adoption', + changes={'batchid': batchid, 'reverted': result['reverted'], + 'conflicts': len(result['conflicts'])}) + db.session.commit() + except ValueError as exc: + db.session.rollback() + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + except Exception: + db.session.rollback() + raise + return success_response(result) + + @settings_bp.route('', methods=['GET']) @jwt_required(optional=True) def list_settings(): diff --git a/shopdb/core/models/__init__.py b/shopdb/core/models/__init__.py index f3d1e732..35eabeb7 100644 --- a/shopdb/core/models/__init__.py +++ b/shopdb/core/models/__init__.py @@ -23,6 +23,8 @@ from .setting import Setting from .auditlog import AuditLog from .customfield import CustomField, CustomFieldValue from .apitoken import ApiToken +from .importbatch import (ImportBatch, ImportJournalEntry, BATCH_APPLIED, + BATCH_UNDONE, OP_UPDATE, OP_INSERT) __all__ = [ # Base @@ -82,4 +84,11 @@ __all__ = [ 'CustomFieldValue', # Personal API tokens 'ApiToken', + # Import batch + journal (reversible bulk imports) + 'ImportBatch', + 'ImportJournalEntry', + 'BATCH_APPLIED', + 'BATCH_UNDONE', + 'OP_UPDATE', + 'OP_INSERT', ] diff --git a/shopdb/core/models/importbatch.py b/shopdb/core/models/importbatch.py new file mode 100644 index 00000000..7eb06da4 --- /dev/null +++ b/shopdb/core/models/importbatch.py @@ -0,0 +1,86 @@ +"""Import batch + journal models (reversible bulk imports). + +PLATFORM-LEVEL, IMPORT-AGNOSTIC. These two tables let any bulk-import writer +record exactly what it changed so the change can be reversed later. Nothing here +knows about Maximo, WINC, machines, or any one source: an importer records +primitive mutations (set column X on row Y, or insert row Y) tied to a batch id, +and a generic undo replays them in reverse. + +importbatches one row per approved write - the identity that approval, the + write, and the eventual undo all reference (the same value + the approval packet calls batchid). +importjournalentries one row per primitive mutation the writer made, ordered so + undo can reverse them (inserts removed after the updates that + pointed at them are reverted). + +The undo contract, kept deliberately conservative: + - update is reverted only if the cell still holds the value we wrote; a value a + human changed since is left as-is and reported (never clobbered). + - insert is hard-deleted, but ONLY the row this batch created and only when + nothing outside the batch still references it. +""" + +from shopdb.extensions import db + + +# Batch lifecycle. +BATCH_APPLIED = 'applied' +BATCH_UNDONE = 'undone' + +# Journal operation kinds. +OP_UPDATE = 'update' +OP_INSERT = 'insert' + + +class ImportBatch(db.Model): + """One approved import write. The batchid is the approval packet's batchid, + carried through write and undo so all three reference one identity.""" + __tablename__ = 'importbatches' + + batchid = db.Column(db.String(36), primary_key=True) + kind = db.Column(db.String(64), nullable=False) + planhash = db.Column(db.String(64), nullable=False) + rowcount = db.Column(db.Integer, nullable=False, default=0) + createdat = db.Column(db.DateTime, nullable=False) + createdby = db.Column(db.String(100), nullable=True) + status = db.Column(db.String(16), nullable=False, default=BATCH_APPLIED) + undoneat = db.Column(db.DateTime, nullable=True) + + entries = db.relationship( + 'ImportJournalEntry', + backref='batch', + cascade='all, delete-orphan', + order_by='ImportJournalEntry.sequence', + ) + + def __repr__(self): + return f"" + + +class ImportJournalEntry(db.Model): + """One reversible primitive mutation made by an import writer.""" + __tablename__ = 'importjournalentries' + + # SQLite only autoincrements a plain INTEGER PRIMARY KEY, not BIGINT, so the + # test database needs the Integer variant while MySQL keeps BIGINT for range. + entryid = db.Column( + db.BigInteger().with_variant(db.Integer, 'sqlite'), + primary_key=True) + batchid = db.Column( + db.String(36), + db.ForeignKey('importbatches.batchid', ondelete='CASCADE'), + nullable=False, + index=True, + ) + sequence = db.Column(db.Integer, nullable=False) + operation = db.Column(db.String(16), nullable=False) + targettable = db.Column(db.String(64), nullable=False) + targetid = db.Column(db.BigInteger, nullable=False) + columnname = db.Column(db.String(64), nullable=True) + oldvalue = db.Column(db.Text, nullable=True) + newvalue = db.Column(db.Text, nullable=True) + isreverted = db.Column(db.Boolean, nullable=False, default=False) + + def __repr__(self): + return (f"") diff --git a/shopdb/core/services/wincimport.py b/shopdb/core/services/wincimport.py new file mode 100644 index 00000000..87464136 --- /dev/null +++ b/shopdb/core/services/wincimport.py @@ -0,0 +1,138 @@ +"""WINC machine-adoption import service (preview / commit / undo / list). + +Web-facing wrapper over the validated WINC CLI helpers. The CLI in +scripts/site_imports/winc drives the same analyzer, writer, and journal; this +module adapts them to a Flask request: it works on the request-scoped +db.session connection (so the route owns the transaction and commit) and returns +plain JSON-serialisable dicts for the API. + +WINC is NOT a source adapter like Azure/Thor. It never creates assets - it +updates existing machines only, requires acknowledgment before creating business +units, and records every change to the reversible import journal. That contract +lives here and in the writer, not in the generic canonical-CSV importer. +""" + +from shopdb.extensions import db +from scripts.site_imports import journal +from scripts.site_imports.approval import build_packet +from scripts.site_imports.winc import writer as winc_writer +from scripts.site_imports.winc.dry_run import analyze_rows, parse_export_text + + +class WincImportError(Exception): + """An uploaded export could not be analyzed or applied as WINC.""" + + +def _connection(): + return db.session.connection() + + +def _plan_and_packet(export_text): + """Parse an uploaded export, build the plan and the approval packet.""" + rows = parse_export_text(export_text) + if not rows: + raise WincImportError('the uploaded file had no rows') + if 'assetnum' not in rows[0]: + raise WincImportError( + 'this does not look like a WINC asset export (no assetnum column)') + + plan, summary, _lookups = analyze_rows(_connection(), rows) + packet = build_packet('winc-machine-adoption', plan, summary, + winc_writer._PACKET_GROUPS) + return plan, summary, packet + + +def _report(summary, packet): + """Shape the packet + summary into the JSON the UI renders.""" + return { + 'kind': packet['kind'], + 'planhash': packet['planhash'], + 'rowcount': packet['rowcount'], + 'summary': dict(summary), + 'needscreate': {group: dict(items) + for group, items in packet.get('needscreate', {}).items()}, + 'unresolved': {group: dict(items) + for group, items in packet.get('unresolved', {}).items()}, + 'blocking': packet.get('blocking', False), + } + + +def preview(export_text): + """Read-only: analyze the export and return the report the UI shows. + + Writes nothing. The whole point is to surface what a commit WOULD do - + counts, the business units it would create (needs acknowledgment), and any + unresolved values that block the import. + """ + _plan, summary, packet = _plan_and_packet(export_text) + report = _report(summary, packet) + report['committed'] = False + return report + + +def commit(export_text, acknowledge=None, createdby=None): + """Apply the plan reversibly inside the request transaction. + + Rebuilds the plan from the same upload, re-checks it against a freshly built + packet (drift, blocking, acknowledgment) and writes through the journal. The + caller owns the commit/rollback of db.session. Returns the report plus the + batchid on success. + + Raises WincImportError (mapped to a 400 by the route) when the plan is + blocking or a needs-create group was not acknowledged. + """ + plan, summary, packet = _plan_and_packet(export_text) + try: + batchid = winc_writer.apply_plan( + _connection(), plan, packet, + acknowledge=acknowledge, createdby=createdby) + except winc_writer.WriterError as exc: + raise WincImportError(str(exc)) + + report = _report(summary, packet) + report['committed'] = True + report['batchid'] = batchid + return report + + +def undo(batchid): + """Reverse an applied batch. Caller owns the transaction. Returns a dict.""" + result = journal.undo_batch(_connection(), batchid, + references=winc_writer._UNDO_REFERENCES) + return { + 'batchid': result.batchid, + 'reverted': result.reverted, + 'conflicts': [ + {'operation': conflict.operation, 'table': conflict.table, + 'targetid': conflict.targetid, 'column': conflict.column, + 'reason': conflict.reason} + for conflict in result.conflicts + ], + } + + +def list_batches(limit=50): + """Applied/undone WINC batches, newest first, for the UI's undo list.""" + from sqlalchemy import text + + def _iso(value): + # PyMySQL hands back datetime objects; SQLite (tests) hands back the + # stored ISO string. Normalise both to an ISO string with a Z suffix. + if value is None: + return None + if hasattr(value, 'isoformat'): + return value.isoformat() + 'Z' + return str(value) + + rows = _connection().execute(text( + 'SELECT batchid, kind, rowcount, createdat, createdby, status, ' + 'undoneat FROM importbatches WHERE kind = :kind ' + 'ORDER BY createdat DESC LIMIT :limit'), + {'kind': 'winc-machine-adoption', 'limit': limit}).fetchall() + return [ + {'batchid': r[0], 'kind': r[1], 'rowcount': r[2], + 'createdat': _iso(r[3]), + 'createdby': r[4], 'status': r[5], + 'undoneat': _iso(r[6])} + for r in rows + ] diff --git a/tests/test_winc_import_routes.py b/tests/test_winc_import_routes.py new file mode 100644 index 00000000..0b273be1 --- /dev/null +++ b/tests/test_winc_import_routes.py @@ -0,0 +1,210 @@ +"""WINC machine-adoption import routes: preview / commit / undo. + +WINC is not a canonical-CSV source adapter (that path is covered by +test_import_sources). It updates existing machines only, needs explicit +acknowledgment before creating a business unit, and every commit is journalled +so it can be undone. These tests drive the real routes end to end against the +in-memory schema: seed a machine asset, upload an export that matches it, and +assert what preview reports, what commit writes, and that undo puts it back. +""" +import io + +import pytest + + +def _seed_machine(db, assetnumber='0083A Debur'): + """Create one machine asset the export can match on, plus a J3 location. + + Returns (assetid, locationid). The workstation key of '0083A Debur' is + '0083A', which the export row 'W0083A' reduces to, so the two crosswalk. + """ + from shopdb.core.models import AssetType, AssetStatus, Location + from shopdb.core.models.asset import Asset + from plugins.machines.models.machine import Machine + + assettype = AssetType(assettype='machine') + status = AssetStatus(status='In Use') + location = Location(locationname='J3') + db.session.add_all([assettype, status, location]) + db.session.flush() + + asset = Asset( + assetnumber=assetnumber, + name='Old Name', + assettypeid=assettype.assettypeid, + statusid=status.statusid, + ) + db.session.add(asset) + db.session.flush() + + db.session.add(Machine(assetid=asset.assetid)) + db.session.commit() + return asset.assetid, location.locationid + + +# An export whose single in-scope row matches the seeded machine. 'WM-J3' +# resolves to the J3 location; 'lg parts' aliases to the 'Large Parts' business +# unit (present or needs-create depending on the test). +EXPORT_CSV = ( + 'assetnum,description,location,eq1\r\n' + 'W0083A,Renamed Machine,WM-J3,lg parts\r\n' + 'FACILITIES-1,Not a machine,,\r\n' +) + + +def _upload(client, headers, url, csv_text=EXPORT_CSV, extra=None): + data = {'files': (io.BytesIO(csv_text.encode('utf-8')), 'winc.csv')} + if extra: + data.update(extra) + return client.post(url, data=data, content_type='multipart/form-data', + headers=headers) + + +PREVIEW = '/api/settings/asset-import/winc/preview' +COMMIT = '/api/settings/asset-import/winc/commit' +UNDO = '/api/settings/asset-import/winc/undo' +BATCHES = '/api/settings/asset-import/winc/batches' + + +def test_preview_reports_plan_without_writing(client, db, auth_headers): + _seed_machine(db) + + response = _upload(client, auth_headers, PREVIEW) + assert response.status_code == 200, response.get_json() + report = response.get_json()['data'] + + # One workstation row in scope, the facilities row filtered out. + assert report['committed'] is False + assert report['rowcount'] == 1 + assert report['summary']['in_scope'] == 1 + assert report['summary']['non_workstation_rows'] == 1 + # 'Large Parts' does not exist yet, so it is flagged for acknowledgment. + assert report['needscreate']['businessunit'] == {'Large Parts': 1} + assert report['blocking'] is False + + # A preview must not have touched the asset. + from shopdb.core.models.asset import Asset + assert Asset.query.first().name == 'Old Name' + + +def test_preview_rejects_non_winc_upload(client, db, auth_headers): + _seed_machine(db) + response = _upload(client, auth_headers, PREVIEW, + csv_text='foo,bar\r\n1,2\r\n') + assert response.status_code == 400 + assert 'assetnum' in response.get_json()['data']['error']['message'] + + +def test_commit_requires_acknowledgment_for_needs_create(client, db, + auth_headers): + _seed_machine(db) + # No acknowledge field: the needs-create business unit blocks the write. + response = _upload(client, auth_headers, COMMIT) + assert response.status_code == 400 + message = response.get_json()['data']['error']['message'].lower() + assert 'acknowledge' in message + + from shopdb.core.models.asset import Asset + from shopdb.core.models import BusinessUnit + assert Asset.query.first().name == 'Old Name' + assert BusinessUnit.query.count() == 0 + + +def test_commit_applies_plan_and_creates_business_unit(client, db, + auth_headers): + assetid, locationid = _seed_machine(db) + + response = _upload(client, auth_headers, COMMIT, + extra={'acknowledge': 'businessunit'}) + assert response.status_code == 200, response.get_json() + report = response.get_json()['data'] + assert report['committed'] is True + assert report['batchid'] + + from shopdb.core.models.asset import Asset + from shopdb.core.models import BusinessUnit + asset = db.session.get(Asset, assetid) + assert asset.name == 'Renamed Machine' + assert asset.locationid == locationid + created = BusinessUnit.query.filter_by(businessunit='Large Parts').one() + assert asset.businessunitid == created.businessunitid + + +def test_commit_uses_existing_business_unit_when_present(client, db, + auth_headers): + from shopdb.core.models import BusinessUnit + existing = BusinessUnit(businessunit='Large Parts') + db.session.add(existing) + db.session.commit() + existing_id = existing.businessunitid + + assetid, _ = _seed_machine(db) + + # Already resolved, so no needs-create and no acknowledgment needed. + response = _upload(client, auth_headers, COMMIT) + assert response.status_code == 200, response.get_json() + + from shopdb.core.models.asset import Asset + asset = db.session.get(Asset, assetid) + assert asset.businessunitid == existing_id + # No duplicate business unit was created. + assert BusinessUnit.query.filter_by(businessunit='Large Parts').count() == 1 + + +def test_undo_reverses_a_committed_batch(client, db, auth_headers): + assetid, _ = _seed_machine(db) + + committed = _upload(client, auth_headers, COMMIT, + extra={'acknowledge': 'businessunit'}) + batchid = committed.get_json()['data']['batchid'] + + from shopdb.core.models.asset import Asset + from shopdb.core.models import BusinessUnit + assert db.session.get(Asset, assetid).name == 'Renamed Machine' + + response = client.post(UNDO, json={'batchid': batchid}, headers=auth_headers) + assert response.status_code == 200, response.get_json() + result = response.get_json()['data'] + assert result['batchid'] == batchid + assert result['reverted'] > 0 + assert result['conflicts'] == [] + + db.session.expire_all() + asset = db.session.get(Asset, assetid) + assert asset.name == 'Old Name' + assert asset.locationid is None + assert asset.businessunitid is None + # The importer-created business unit was hard-deleted on undo. + assert BusinessUnit.query.filter_by(businessunit='Large Parts').count() == 0 + + +def test_undo_requires_batchid(client, db, auth_headers): + response = client.post(UNDO, json={}, headers=auth_headers) + assert response.status_code == 400 + assert 'batchid' in response.get_json()['data']['error']['message'].lower() + + +def test_batches_lists_committed_then_undone(client, db, auth_headers): + _seed_machine(db) + committed = _upload(client, auth_headers, COMMIT, + extra={'acknowledge': 'businessunit'}) + batchid = committed.get_json()['data']['batchid'] + + listed = client.get(BATCHES, headers=auth_headers).get_json()['data'] + batch = next(b for b in listed['batches'] if b['batchid'] == batchid) + assert batch['status'] == 'applied' + assert batch['kind'] == 'winc-machine-adoption' + + client.post(UNDO, json={'batchid': batchid}, headers=auth_headers) + listed = client.get(BATCHES, headers=auth_headers).get_json()['data'] + batch = next(b for b in listed['batches'] if b['batchid'] == batchid) + assert batch['status'] == 'undone' + assert batch['undoneat'] is not None + + +def test_winc_routes_require_admin(client, db, member_headers): + _seed_machine(db) + for url in (PREVIEW, COMMIT): + response = _upload(client, member_headers, url) + assert response.status_code in (401, 403) + assert client.get(BATCHES, headers=member_headers).status_code in (401, 403)