Catalog seeding, blank-unique fix, slides rotation, console repair

This commit is contained in:
2026-08-05 13:19:51 -04:00
parent 9c530a70db
commit d3e7311376
13 changed files with 3721 additions and 219 deletions

View File

@@ -201,6 +201,7 @@ var
// the check takes a few seconds, runs hidden, and gives no sign of life.
CheckingPage: TOutputProgressWizardPage;
PluginPage: TInputOptionWizardPage;
CatalogPage: TInputOptionWizardPage;
PluginNames: TArrayOfString;
PluginPageReady: Boolean;
DbPageReady: Boolean;
@@ -397,7 +398,21 @@ begin
if Trim(PluginNames[I]) <> '' then
PluginPage.Add(PluginLabel(Trim(PluginNames[I])));
DbChoicePage := CreateInputOptionPage(PluginPage.ID,
// Offered, not forced. A site that machines nothing inherits 21 machine types
// it will never use, and a catalog nobody wanted is just clutter in every
// dropdown. Ticked by default because the alternative is retyping a vendor and
// model list another site already built.
CatalogPage := CreateInputOptionPage(PluginPage.ID,
'Starter data', 'Load the shared equipment catalog?',
'Vendors, model numbers and the type lists, as built up across the other ' +
'sites. It only ADDS - nothing existing is changed or removed - and it can ' +
'be loaded later with "flask seed catalog". No assets, locations or people ' +
'are included; this is catalogue information only.',
False, False);
CatalogPage.Add('Load the shared vendor and model catalog');
CatalogPage.Values[0] := True;
DbChoicePage := CreateInputOptionPage(CatalogPage.ID,
'Database', 'Where should ShopDB-Flask store its data?',
'Most sites already run MySQL for the existing shopdb application. If so, ' +
'choose the second option - installing a second server would collide on ' +
@@ -1177,6 +1192,8 @@ begin
' -OnFailure never' +
' -ClientIpSource ' + ClientIpSourceArg +
' -SitePlugins "' + SelectedPlugins + '"';
if CatalogPage.Values[0] then
Common := Common + ' -SeedCatalog';
// Subpath deployment: an IIS Application under the existing site instead of a
// site of its own. Appended here because Pascal has no conditional expression.

View File

@@ -22,7 +22,7 @@
[CmdletBinding()]
param(
[ValidateSet('menu','status','start','stop','restart','logs','open','backup',
'check','sessions','plugins','add-plugin','verify','uninstall')]
'check','repair','sessions','plugins','add-plugin','verify','uninstall')]
[string] $Command = 'menu',
[string] $Path = '',
# Machine-readable output for 'check'. The people running this are expected to
@@ -577,11 +577,101 @@ function Invoke-Check {
$env:FLASK_APP = 'shopdb'
try { & $flask db-utils preflight 2>&1 | ForEach-Object { Say " $_" } }
finally { Pop-Location }
# Surface an unfinished install HERE, where somebody is already looking,
# rather than leaving them to infer it from 500s on unrelated pages.
$state = Test-Provisioned
if ($state.SchemaCurrent -eq $false -or $state.Seeded -eq $false) {
Say ''
Say ' THIS SERVER IS NOT FULLY PROVISIONED' 'Red'
foreach ($detail in $state.Detail) { Say (" - {0}" -f $detail) 'Red' }
Say ' Most pages will answer 500 until this is fixed. Run:' 'Yellow'
Say ' shopdb-admin.ps1 repair' 'White'
}
Say ''
Say ' For help from an AI assistant, paste the output of:' 'DarkGray'
Say ' shopdb-admin.ps1 check -Json' 'White'
}
function Test-Provisioned {
"""Is the schema current and the reference data seeded?"""
# A server whose migrations or seeds never completed does not fail politely:
# it answers 500 on most pages and 404 on settings that were never created,
# which reads as a broken application rather than an unfinished install. One
# site was stood up that way and spent a morning being debugged as a bug.
$result = @{ SchemaCurrent = $true; Seeded = $true; Detail = @() }
$out = Invoke-Flask @('db','current')
$current = ($out | Out-String)
$head = (Invoke-Flask @('db','heads') | Out-String)
if ($script:LastFlaskExit -ne 0) {
$result.SchemaCurrent = $false
$result.Detail += 'could not read the schema version'
} elseif ($current -notmatch '\(head\)' -and $head.Trim()) {
# `db current` appends "(head)" when the database is at the newest
# revision. Its absence means migrations are outstanding.
$result.SchemaCurrent = $false
$result.Detail += 'database schema is behind the application'
}
# Sentinel seeds. permissions, settings and the reference data are each
# created by a `flask seed` command, and their absence is what produced the
# 404s on settings keys at a site whose install never finished.
$seedOut = (Invoke-Flask @('db-utils','seed-state') | Out-String)
if ($seedOut -match 'No such command') {
# An older build with no seed-state. Report UNKNOWN rather than healthy:
# claiming a clean bill of health from a check that did not run is how a
# broken server passes inspection.
$result.Seeded = $null
$result.Detail += 'seed state could not be checked (older build)'
} elseif ($script:LastFlaskExit -ne 0 -or $seedOut -match 'MISSING') {
$result.Seeded = $false
$result.Detail += 'seed data is missing (permissions, settings or reference data)'
}
return $result
}
function Invoke-Repair {
Head 'Repair provisioning'
Say ' Brings the database up to the application: migrations, plugin'
Say ' migrations, and the seed data. Every step is idempotent, so running'
Say ' this on a healthy server changes nothing.'
Say ''
$steps = @(
@{ Name = 'core schema'; Args = @('db','upgrade') },
@{ Name = 'plugin schemas'; Args = @('plugin','upgrade-all') },
@{ Name = 'permissions'; Args = @('seed','permissions') },
@{ Name = 'settings'; Args = @('seed','settings') },
@{ Name = 'reference data'; Args = @('seed','reference-data') }
)
$failed = @()
foreach ($step in $steps) {
Say (" {0} ..." -f $step.Name) 'White'
$out = Invoke-Flask $step.Args
if ($script:LastFlaskExit -ne 0) {
$failed += $step.Name
Say (" FAILED (exit {0})" -f $script:LastFlaskExit) 'Red'
$out | Select-Object -Last 6 | ForEach-Object { Say " $_" 'Red' }
} else {
Say ' done' 'Green'
}
}
if ($failed.Count -gt 0) {
Say ''
Say (" {0} step(s) failed: {1}" -f $failed.Count, ($failed -join ', ')) 'Red'
Say ' Nothing further was skipped - each step ran independently.' 'Red'
Say ' Send the output above, or run: shopdb-admin.ps1 check -Json' 'Red'
return
}
Say ''
Say ' Provisioning complete. Restarting the application.' 'Green'
Restart-App
}
function Show-Sessions {
Head 'Worker processes'
$w = Get-CimInstance Win32_Process -Filter "Name='w3wp.exe'" -EA SilentlyContinue
@@ -856,6 +946,7 @@ switch ($Command) {
'open' { Open-Site }
'backup' { Backup-Db $Path }
'check' { Invoke-Check }
'repair' { Invoke-Repair }
'sessions' { Show-Sessions }
'plugins' { Show-Plugins }
'add-plugin'{ Add-Plugin $Path }

View File

@@ -127,6 +127,10 @@ param(
# client reads as 127.0.0.1: the GE-Enforce IP allowlist, the dashboard
# visitor-location lookup and per-host login rate limiting all break quietly.
[ValidateSet('direct','proxy')] [string] $ClientIpSource = 'direct',
# Load the shared vendor/model catalog during stage 3. Offered by the
# wizard rather than assumed: a site that machines nothing does not want
# 21 machine types cluttering its dropdowns.
[switch] $SeedCatalog,
# Required before this installer will alter an installation it did not
# create. Everything here is built for a greenfield server: it makes its own
# Python, its own venv and its own IIS objects, and its upgrade path assumes
@@ -1747,6 +1751,14 @@ build to get a matching pair, then send the install log to support.
Invoke-Native $Flask @('seed',$seed) "seed $seed"
}
# Additive and idempotent, so a re-run adds only what is genuinely new
# and never overwrites a description somebody corrected here.
if ($SeedCatalog) {
Write-Log 'flask seed catalog (shared vendor and model catalog)'
$catalogOut = Invoke-Native $Flask @('seed','catalog') 'seed catalog' -OkExit 0,1
$catalogOut | Select-Object -Last 12 | ForEach-Object { Write-Log " $_" }
}
# Plugin registry starts empty on a fresh box. apply-profile resolves the
# declared set's hard-dependency closure and installs + enables it in
# dependency order, idempotently (ADR-013). It fails loudly if the profile

View File

@@ -205,30 +205,41 @@
<!-- Map Position (optional) -->
<fieldset>
<legend>Map Position (Optional)</legend>
<div class="form-row">
<div class="form-group">
<label for="mapx">Map X Position</label>
<input
id="mapx"
v-model.number="form.mapx"
type="number"
class="form-control"
min="0"
/>
</div>
<div class="form-group">
<label for="mapy">Map Y Position</label>
<input
id="mapy"
v-model.number="form.mapy"
type="number"
class="form-control"
min="0"
/>
<!--
Picked on the floor plan, the same way machines, PCs and printers do
it. This form used to ask for the X and Y as raw numbers, which meant
reading coordinates off another screen and typing them in.
-->
<div class="form-group">
<label>Map Location</label>
<div class="map-location-control">
<div v-if="form.mapx !== null && form.mapy !== null" class="current-position">
Position: {{ form.mapx }}, {{ form.mapy }}
<button type="button" class="btn btn-sm btn-secondary" @click="clearMapPosition">Clear</button>
</div>
<button type="button" class="btn btn-secondary" @click="showMapPicker = true">
Set Location on Map
</button>
</div>
</div>
</fieldset>
<!-- Map Picker Modal -->
<Modal v-model="showMapPicker" title="Select Location on Map" size="fullscreen">
<div class="map-modal-content">
<ShopFloorMap
:pickerMode="true"
:initialPosition="form.mapx !== null ? { left: form.mapx, top: form.mapy } : null"
:theme="currentTheme"
@positionPicked="handlePositionPicked"
/>
</div>
<template #footer>
<button class="btn btn-secondary" @click="showMapPicker = false">Cancel</button>
<button class="btn btn-primary" @click="confirmMapPosition">Confirm Location</button>
</template>
</Modal>
<!-- Notes -->
<fieldset>
<legend>Notes</legend>
@@ -274,11 +285,16 @@ import {
businessunitsApi
} from '@/api'
import CustomFieldsInputs from '@/components/CustomFieldsInputs.vue'
import ShopFloorMap from '@/components/ShopFloorMap.vue'
import Modal from '@/components/Modal.vue'
import { currentTheme } from '@/stores/theme'
import { useIdentifierFlags } from '@/composables/identifierSettings'
import { apiError } from '@/utils/apiError'
const { isEnabled } = useIdentifierFlags()
const showMapPicker = ref(false)
const tempMapPosition = ref(null)
const route = useRoute()
const router = useRouter()
@@ -483,6 +499,24 @@ function cancel() {
router.push('/network')
}
}
function handlePositionPicked(position) {
tempMapPosition.value = position
}
function confirmMapPosition() {
if (tempMapPosition.value) {
form.value.mapx = tempMapPosition.value.left
form.value.mapy = tempMapPosition.value.top
}
showMapPicker.value = false
}
function clearMapPosition() {
form.value.mapx = null
form.value.mapy = null
tempMapPosition.value = null
}
</script>
<style scoped>

View File

@@ -18,7 +18,7 @@
<!-- The tag prefix is shown as a fixed addon so operators type only
the number part off the label. -->
<div class="entry-input-group">
<span class="entry-prefix">{{ LABEL_PREFIX }}</span>
<span v-if="LABEL_PREFIX" class="entry-prefix">{{ LABEL_PREFIX }}</span>
<input ref="entryInput" v-model="manualCode" class="entry-input has-prefix"
inputmode="none" autocomplete="off" placeholder="number"
@keydown.enter.prevent="lookupItem(manualCode)" />
@@ -92,13 +92,16 @@
<script setup>
import { ref, onMounted, onBeforeUnmount, nextTick } from 'vue'
import { printedpartsApi } from '@/api'
import { printedpartsApi, settingsApi } from '@/api'
import { withBase } from '@/utils/basePath'
import TouchKeypad from '@/components/TouchKeypad.vue'
// Fixed tag prefix shown before the label number at the kiosk. Change here if
// the gage lab tags use a different leading string.
const LABEL_PREFIX = 'WJ'
// Leading text on the physical labels, shown before the number box so operators
// type only the digits. It was hardcoded to 'WJ', which is West Jefferson's tag
// format and nobody else's - on another site's kiosk it told operators to expect
// letters that are not on their labels. Display only: the lookup sends whatever
// was typed, unprefixed.
const LABEL_PREFIX = ref('')
const step = ref('item')
const item = ref(null)
@@ -113,7 +116,20 @@ const scannedRevision = ref(null)
const entryInput = ref(null)
let resetTimer = null
onMounted(focusEntry)
onMounted(async () => {
// A kiosk left running for weeks reads this once at load, which is fine: the
// label format is a property of the site's physical tags, not something that
// changes during a shift.
try {
const response = await settingsApi.get('printedparts_label_prefix')
LABEL_PREFIX.value = (response.data?.data?.value || '').trim()
} catch (err) {
// Never block the kiosk over a cosmetic hint. An unreachable or unseeded
// setting simply means no prefix is shown.
LABEL_PREFIX.value = ''
}
focusEntry()
})
onBeforeUnmount(() => clearTimeout(resetTimer))
function focusEntry(event) {

View File

@@ -18,6 +18,17 @@
</p>
</div>
<div class="form-group">
<label>Kiosk label prefix</label>
<input v-model="values.printedparts_label_prefix" type="text"
class="form-control" maxlength="8" placeholder="none" />
<p class="field-hint">
Leading text on this site's physical labels, shown at the kiosk before the
number box so operators type only the digits. Leave it empty to show no
prefix. This is about the printed tags, not the item codes above.
</p>
</div>
<div class="form-group">
<label>Default low-stock threshold</label>
<input v-model.number="values.printedparts_default_threshold"
@@ -103,6 +114,7 @@ import { settingsApi, usersApi, supportteamsApi } from '@/api'
const KEYS = [
'printedparts_code_prefix',
'printedparts_label_prefix',
'printedparts_default_threshold',
'printedparts_unknown_badge',
'printedparts_alert_email',
@@ -113,6 +125,7 @@ const KEYS = [
const values = ref({
printedparts_code_prefix: '3DP',
printedparts_label_prefix: '',
printedparts_default_threshold: 5,
printedparts_unknown_badge: 'deny',
printedparts_alert_email: '',

View File

@@ -126,6 +126,10 @@ class PrintedpartsPlugin(BasePlugin):
defaults = [
('printedparts_code_prefix', '3DP', 'string',
'Prefix for generated item codes'),
('printedparts_label_prefix', '', 'string',
'Leading text on the physical gage-lab labels, shown at the kiosk '
'before the number box so operators type only the digits. Empty '
'shows no prefix. Site-specific: West Jefferson labels read WJ'),
('printedparts_default_threshold', '5', 'integer',
'Default low-stock threshold for new items'),
('printedparts_unknown_badge', 'deny', 'string',

View File

@@ -1,191 +1,207 @@
<template>
<div class="tv-dashboard">
<div class="slideshow-container">
<div
v-for="(slide, idx) in slides"
:key="slide.filename"
class="slide"
:class="{ active: idx === currentSlide }"
>
<img :src="withBase(basePath + slide.filename)" :alt="slide.filename" />
</div>
<div v-if="error" class="error-message">
<h2>Display Error</h2>
<p>{{ error }}</p>
</div>
<div v-if="!slides.length && !error" class="error-message">
<h2>No Slides</h2>
<p>No slides configured for display</p>
</div>
</div>
<div class="progress-bar" :style="progressStyle"></div>
</div>
</template>
<script setup>
import { ref, computed, onMounted, onUnmounted } from 'vue'
import { useRoute } from 'vue-router'
import api from '@/api'
import { withBase } from '@/utils/basePath'
// Which slide surface this display renders: 'lobby' (default) or 'shopfloor'
// (the screensaver). Set by the route meta (/screensaver) or a ?surface= query.
const route = useRoute()
const VALID_SURFACES = ['lobby', 'shopfloor']
const surface = (VALID_SURFACES.includes(route.meta?.surface) && route.meta.surface)
|| (VALID_SURFACES.includes(route.query?.surface) && route.query.surface)
|| 'lobby'
const INTERVAL = 10 // seconds between slides
const slides = ref([])
const basePath = ref('/static/slides/')
const currentSlide = ref(0)
const error = ref('')
const progress = ref(0)
let slideTimer = null
let progressTimer = null
let refreshTimer = null
const progressStyle = computed(() => ({
width: `${progress.value}%`,
transition: progress.value === 0 ? 'none' : `width ${INTERVAL}s linear`
}))
onMounted(async () => {
await fetchSlides()
// Start slideshow if we have slides
if (slides.value.length > 1) {
startSlideshow()
}
// Refresh slide list every 60 seconds
refreshTimer = setInterval(fetchSlides, 60000)
})
onUnmounted(() => {
if (slideTimer) clearTimeout(slideTimer)
if (progressTimer) clearTimeout(progressTimer)
if (refreshTimer) clearInterval(refreshTimer)
})
async function fetchSlides() {
try {
// Flat feed shape: { success, surface, basepath, interval, slides:[...] }
const response = await api.get('/slides/feed', { params: { surface } })
const data = response.data
if (data.success && data.slides?.length > 0) {
slides.value = data.slides
basePath.value = data.basepath || `/api/slides/img/${surface}/`
error.value = ''
// Restart slideshow if slides changed
if (slides.value.length > 1 && !slideTimer) {
startSlideshow()
}
} else {
slides.value = []
error.value = 'No slides configured'
}
} catch (err) {
console.error('Error fetching slides:', err)
error.value = 'Unable to load slides'
}
}
function startSlideshow() {
scheduleNextSlide()
}
function scheduleNextSlide() {
// Reset progress bar
progress.value = 0
// Start progress animation after a small delay
progressTimer = setTimeout(() => {
progress.value = 100
}, 50)
// Schedule next slide
slideTimer = setTimeout(() => {
nextSlide()
scheduleNextSlide()
}, INTERVAL * 1000)
}
function nextSlide() {
if (slides.value.length === 0) return
currentSlide.value = (currentSlide.value + 1) % slides.value.length
}
</script>
<style scoped>
.tv-dashboard {
background: #000;
width: 100vw;
height: 100vh;
margin: 0;
padding: 0;
overflow: hidden;
}
.slideshow-container {
position: relative;
width: 100%;
height: 100%;
}
.slide {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
opacity: 0;
transition: opacity 1s ease-in-out;
display: flex;
align-items: center;
justify-content: center;
}
.slide.active {
opacity: 1;
}
.slide img {
width: 100%;
height: 100%;
object-fit: contain;
}
.progress-bar {
position: fixed;
bottom: 0;
left: 0;
height: 4px;
background: #4181ff;
z-index: 100;
}
.error-message {
position: fixed;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
text-align: center;
color: #fff;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
font-size: 24px;
}
.error-message h2 {
color: #ff4444;
margin-bottom: 20px;
}
</style>
<template>
<div class="tv-dashboard">
<div class="slideshow-container">
<div
v-for="(slide, idx) in slides"
:key="slide.filename"
class="slide"
:class="{ active: idx === currentSlide }"
>
<img :src="withBase(basePath + slide.filename)" :alt="slide.filename" />
</div>
<div v-if="error" class="error-message">
<h2>Display Error</h2>
<p>{{ error }}</p>
</div>
<div v-if="!slides.length && !error" class="error-message">
<h2>No Slides</h2>
<p>No slides configured for display</p>
</div>
</div>
<div class="progress-bar" :style="progressStyle"></div>
</div>
</template>
<script setup>
import { ref, computed, onMounted, onUnmounted } from 'vue'
import { useRoute } from 'vue-router'
import api from '@/api'
import { withBase } from '@/utils/basePath'
// Which slide surface this display renders: 'lobby' (default) or 'shopfloor'
// (the screensaver). Set by the route meta (/screensaver) or a ?surface= query.
const route = useRoute()
const VALID_SURFACES = ['lobby', 'shopfloor']
const surface = (VALID_SURFACES.includes(route.meta?.surface) && route.meta.surface)
|| (VALID_SURFACES.includes(route.query?.surface) && route.query.surface)
|| 'lobby'
const INTERVAL = 10 // seconds between slides
const slides = ref([])
const basePath = ref('/static/slides/')
const currentSlide = ref(0)
const error = ref('')
const progress = ref(0)
let slideTimer = null
let progressTimer = null
let refreshTimer = null
const progressStyle = computed(() => ({
width: `${progress.value}%`,
// Matches the slide's own duration, or the bar finishes early and sits full.
transition: progress.value === 0 ? 'none' : `width ${currentSlideSeconds()}s linear`
}))
onMounted(async () => {
// fetchSlides starts the slideshow itself once it has more than one slide.
// Starting it again here ran TWO timer chains at once, and the second
// assignment to slideTimer lost the handle to the first so nothing could stop
// it. Both chains called nextSlide a few milliseconds apart, so with exactly
// two slides the display advanced 0 -> 1 -> 0 every cycle and looked frozen.
await fetchSlides()
// Refresh slide list every 60 seconds
refreshTimer = setInterval(fetchSlides, 60000)
})
onUnmounted(() => {
if (slideTimer) clearTimeout(slideTimer)
if (progressTimer) clearTimeout(progressTimer)
if (refreshTimer) clearInterval(refreshTimer)
})
async function fetchSlides() {
try {
// Flat feed shape: { success, surface, basepath, interval, slides:[...] }
const response = await api.get('/slides/feed', { params: { surface } })
const data = response.data
if (data.success && data.slides?.length > 0) {
slides.value = data.slides
basePath.value = data.basepath || `/api/slides/img/${surface}/`
error.value = ''
// Restart slideshow if slides changed
if (slides.value.length > 1 && !slideTimer) {
startSlideshow()
}
} else {
slides.value = []
error.value = 'No slides configured'
}
} catch (err) {
console.error('Error fetching slides:', err)
error.value = 'Unable to load slides'
}
}
function startSlideshow() {
scheduleNextSlide()
}
function scheduleNextSlide() {
// Cancel anything already pending. Belt and braces after the double-start
// bug above: any future caller that schedules twice now replaces the timer
// rather than leaking an untracked one.
if (slideTimer) { clearTimeout(slideTimer); slideTimer = null }
if (progressTimer) { clearTimeout(progressTimer); progressTimer = null }
// Reset progress bar
progress.value = 0
// Start progress animation after a small delay
progressTimer = setTimeout(() => {
progress.value = 100
}, 50)
// The feed sends a per-slide duration and this used to ignore it, so a slide
// set to hold for a minute still changed after ten seconds.
const seconds = currentSlideSeconds()
slideTimer = setTimeout(() => {
nextSlide()
scheduleNextSlide()
}, seconds * 1000)
}
function currentSlideSeconds() {
const slide = slides.value[currentSlide.value]
const seconds = Number(slide && slide.seconds)
return seconds > 0 ? seconds : INTERVAL
}
function nextSlide() {
if (slides.value.length === 0) return
currentSlide.value = (currentSlide.value + 1) % slides.value.length
}
</script>
<style scoped>
.tv-dashboard {
background: #000;
width: 100vw;
height: 100vh;
margin: 0;
padding: 0;
overflow: hidden;
}
.slideshow-container {
position: relative;
width: 100%;
height: 100%;
}
.slide {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
opacity: 0;
transition: opacity 1s ease-in-out;
display: flex;
align-items: center;
justify-content: center;
}
.slide.active {
opacity: 1;
}
.slide img {
width: 100%;
height: 100%;
object-fit: contain;
}
.progress-bar {
position: fixed;
bottom: 0;
left: 0;
height: 4px;
background: #4181ff;
z-index: 100;
}
.error-message {
position: fixed;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
text-align: center;
color: #fff;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
font-size: 24px;
}
.error-message h2 {
color: #ff4444;
margin-bottom: 20px;
}
</style>

173
scripts/export_catalog.py Normal file
View File

@@ -0,0 +1,173 @@
"""Export the reference CATALOG from a live instance to a portable JSON file.
A new site starts with almost nothing: `flask seed reference-data` writes a
dozen generic model types and no vendors or models at all. So the first thing
anyone does at a new facility is retype a vendor and model list that another
facility already spent a year building. This lets that catalog travel.
CATALOG ONLY. Vendors, models and the type vocabularies are descriptions of
equipment that exists in the world, and are the same at every site. Assets,
locations, employees, business units and anything with a serial number are that
site's own records and are deliberately NOT exported - nobody wants West
Jefferson's machines appearing at another plant.
Vendor contact details are also skipped: a rep's name and phone number belong to
whoever has the relationship, not to the catalog.
python scripts/export_catalog.py # to shopdb/data/catalog.json
python scripts/export_catalog.py --out mine.json
python scripts/export_catalog.py --pretty # readable diffs in git
Load it on another instance with: flask seed catalog
"""
import argparse
import json
import os
import sys
from collections import OrderedDict
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
DEFAULT_OUT = os.path.join('shopdb', 'data', 'catalog.json')
def export(connection):
from sqlalchemy import inspect, text
inspector = inspect(connection)
tables = set(inspector.get_table_names())
data = OrderedDict()
def rows(sql):
return [dict(r._mapping) for r in connection.execute(text(sql))]
def has(table):
return table in tables
# --- type vocabularies -------------------------------------------------
# Each is a small controlled list, and a site with none of them cannot even
# classify what it owns.
if has('modeltypes'):
data['modeltypes'] = rows("""
SELECT modeltype, category, description, icon
FROM modeltypes ORDER BY modeltype""")
for table, column in (('machinetypes', 'machinetype'),
('computertypes', 'computertype'),
('printertypes', 'printertype'),
('networkdevicetypes', 'networkdevicetype'),
('locationtypes', 'locationtype')):
if has(table):
extra = ', color' if 'color' in {c['name'] for c in inspector.get_columns(table)} else ''
data[table] = rows('SELECT %s, description%s FROM %s ORDER BY %s'
% (column, extra, table, column))
if has('operatingsystems'):
columns = {c['name'] for c in inspector.get_columns('operatingsystems')}
wanted = [c for c in ('osname', 'osversion', 'osfamily', 'description') if c in columns]
data['operatingsystems'] = rows('SELECT %s FROM operatingsystems ORDER BY osname'
% ', '.join(wanted))
# --- vendors -----------------------------------------------------------
# Name and website only. Contact person, phone and address are a site's own
# relationship with that vendor, not a property of the vendor.
if has('vendors'):
columns = {c['name'] for c in inspector.get_columns('vendors')}
wanted = [c for c in ('vendor', 'website', 'description') if c in columns]
data['vendors'] = rows('SELECT %s FROM vendors ORDER BY vendor' % ', '.join(wanted))
# --- models ------------------------------------------------------------
# Referenced by NAME, not id: ids differ between databases, and a catalog
# keyed on them would silently attach models to the wrong vendor.
if has('models'):
data['models'] = rows("""
SELECT m.modelnumber AS modelnumber,
v.vendor AS vendor,
mt.modeltype AS modeltype,
m.description AS description,
m.documentationurl AS documentationurl,
m.imageurl AS imageurl
FROM models m
LEFT JOIN vendors v ON m.vendorid = v.vendorid
LEFT JOIN modeltypes mt ON m.modeltypeid = mt.modeltypeid
ORDER BY v.vendor, m.modelnumber""")
# --- printer supplies --------------------------------------------------
# The single most useful thing to inherit after the models themselves: a new
# site with printers gets every toner, drum and maintenance-kit part number
# already matched to the right model, instead of somebody reading them off
# cartridges. Keyed by the model's natural key for the same reason models
# are keyed by vendor name.
if has('modelsupplies') and has('models'):
data['modelsupplies'] = rows("""
SELECT mo.modelnumber AS modelnumber,
v.vendor AS vendor,
s.supplytype AS supplytype,
s.color AS color,
s.capacitytier AS capacitytier,
s.partnumber AS partnumber,
s.marketingname AS marketingname,
s.pageyield AS pageyield,
s.notes AS notes
FROM modelsupplies s
JOIN models mo ON s.modelnumberid = mo.modelnumberid
LEFT JOIN vendors v ON mo.vendorid = v.vendorid
ORDER BY v.vendor, mo.modelnumber, s.partnumber""")
# --- small generic vocabularies ---------------------------------------
# Each is a controlled list that means the same thing at any site.
if has('measuringtooltypes'):
data['measuringtooltypes'] = rows(
'SELECT name, description, color FROM measuringtooltypes ORDER BY name')
if has('notificationtypes'):
data['notificationtypes'] = rows(
'SELECT typename, typedescription, typecolor FROM notificationtypes ORDER BY typename')
if has('accessprotocols'):
columns = {c['name'] for c in inspector.get_columns('accessprotocols')}
wanted = [c for c in ('name', 'scheme', 'defaultport', 'linktemplate') if c in columns]
data['accessprotocols'] = rows('SELECT %s FROM accessprotocols ORDER BY name'
% ', '.join(wanted))
# Drop empty optional values so the file stays readable and a NULL does not
# arrive as the string "None".
for key, records in data.items():
cleaned = []
for record in records:
cleaned.append({k: v for k, v in record.items() if v not in (None, '')})
data[key] = cleaned
return data
def main():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--out', default=DEFAULT_OUT, help='output file')
parser.add_argument('--pretty', action='store_true',
help='indent the JSON so git diffs are readable')
args = parser.parse_args()
from shopdb import create_app
from shopdb.extensions import db
app = create_app()
with app.app_context():
data = export(db.session.connection())
directory = os.path.dirname(os.path.abspath(args.out))
if directory:
os.makedirs(directory, exist_ok=True)
with open(args.out, 'w', encoding='utf-8') as handle:
json.dump(data, handle, indent=2 if args.pretty else None,
ensure_ascii=False, sort_keys=False)
handle.write('\n')
print('Wrote %s' % args.out)
for key, records in data.items():
print(' %-22s %d' % (key, len(records)))
print('')
print('Load it on another instance with: flask seed catalog')
return 0
if __name__ == '__main__':
sys.exit(main())

View File

@@ -88,6 +88,12 @@ def create_app(config_name: str = None) -> Flask:
# Initialize extensions
init_extensions(app)
# An optional unique column accepts any number of NULLs but exactly one
# empty string, so a second blank code collided and became a 500. Normalise
# blank to NULL once, at the mapper, rather than in each endpoint.
from .utils.blankunique import register_blank_unique_normaliser
register_blank_unique_normaliser()
# Initialize plugin manager
with app.app_context():
plugin_manager.init_app(app, db)
@@ -186,6 +192,31 @@ def register_error_handlers(app: Flask):
from .utils.responses import error_response, ErrorCodes
from .exceptions import ShopDBException
# A uniqueness violation is the caller's problem, not a server fault. Without
# this it surfaced as a bare 500 with a SQLAlchemy traceback in the log and
# nothing usable on screen - the operator saw "internal server error" for
# having reused a code that was already taken.
from sqlalchemy.exc import IntegrityError
@app.errorhandler(IntegrityError)
def handle_integrity_error(error):
from .extensions import db
db.session.rollback()
message = str(getattr(error, 'orig', error))
app.logger.warning('integrity error: %s', message)
if 'Duplicate entry' in message or 'UNIQUE constraint' in message:
return error_response(
ErrorCodes.CONFLICT,
'That value is already in use. Codes and identifiers must be unique.',
http_code=409)
if 'foreign key constraint' in message.lower():
return error_response(
ErrorCodes.VALIDATION_ERROR,
'That record refers to something that does not exist, or is still in use elsewhere.',
http_code=400)
return error_response(ErrorCodes.VALIDATION_ERROR,
'The database rejected that change.', http_code=400)
@app.errorhandler(ShopDBException)
def handle_shopdb_exception(error):
http_codes = {

View File

@@ -10,6 +10,58 @@ def db_cli():
pass
@db_cli.command('seed-state')
@with_appcontext
def seed_state():
"""Report whether the seed data a working install depends on is present.
A server whose seeds never ran does not fail politely: settings endpoints
answer 404 for keys that were never created and most pages answer 500, which
reads as a broken application rather than an unfinished install. This gives
the operator console something definite to test, so it can say "run repair"
instead of leaving somebody to infer it from unrelated errors.
Prints one line per group and exits non-zero if anything is missing, so it
can be used as a gate as well as read by a person.
"""
import sys
from shopdb.extensions import db
from sqlalchemy import text
# Sentinels, not exhaustive counts. Each is created by one of the three seed
# commands, so a zero here means that command never ran.
checks = [
('permissions', 'SELECT COUNT(*) FROM permissions', 'flask seed permissions'),
('settings', 'SELECT COUNT(*) FROM settings', 'flask seed settings'),
('asset types', 'SELECT COUNT(*) FROM assettypes', 'flask seed reference-data'),
('location types', 'SELECT COUNT(*) FROM locationtypes', 'flask seed reference-data'),
]
missing = []
for label, sql, remedy in checks:
try:
count = db.session.execute(text(sql)).scalar() or 0
except Exception as exc:
click.echo(click.style(' MISSING ', fg='red') +
'%s - table unreadable (%s)' % (label, type(exc).__name__))
missing.append((label, remedy))
continue
if count == 0:
click.echo(click.style(' MISSING ', fg='red') +
'%s - none present, run: %s' % (label, remedy))
missing.append((label, remedy))
else:
click.echo(click.style(' OK ', fg='green') + '%s (%d)' % (label, count))
if missing:
click.echo('')
click.echo(click.style('%d group(s) missing. This server is not fully provisioned.'
% len(missing), fg='red'))
sys.exit(1)
click.echo('')
click.echo(click.style('Seed data present.', fg='green'))
@db_cli.command('create-all')
@with_appcontext
def create_all():
@@ -147,6 +199,189 @@ def seed_cli():
pass
@seed_cli.command('catalog')
@click.option('--file', 'path', default=None,
help='catalog JSON to load (default shopdb/data/catalog.json)')
@click.option('--dry-run', is_flag=True, help='report what would be added, write nothing')
@with_appcontext
def seed_catalog(path, dry_run):
"""Load the shared vendor/model catalog shipped with the product.
`seed reference-data` writes a dozen generic model types and no vendors or
models, so every new site began by retyping a catalog another site had
already built. This loads that catalog instead.
IDEMPOTENT and ADDITIVE. Records are matched by natural key - a vendor by
name, a model by vendor plus model number, a type by its name - so running
it twice adds nothing the second time. It never updates or deletes an
existing record: a site that has corrected a description or pointed a model
at its own photo keeps its version.
Catalog only. Nothing here identifies a site: no assets, locations,
employees or serial numbers.
"""
import json
import os
from shopdb.extensions import db
from shopdb.core.models import ModelType, OperatingSystem, LocationType, Vendor, Model
if not path:
path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'data', 'catalog.json')
if not os.path.isfile(path):
click.echo(click.style('No catalog file at %s' % path, fg='red'))
raise SystemExit(1)
with open(path, encoding='utf-8') as handle:
data = json.load(handle)
added = {}
def note(kind, count):
if count:
added[kind] = added.get(kind, 0) + count
def simple(key, model, column):
"""Type vocabularies: one row per name."""
records = data.get(key) or []
count = 0
for record in records:
name = record.get(column)
if not name:
continue
if db.session.query(model).filter_by(**{column: name}).first():
continue
db.session.add(model(**{k: v for k, v in record.items()
if hasattr(model, k)}))
count += 1
note(key, count)
simple('modeltypes', ModelType, 'modeltype')
simple('locationtypes', LocationType, 'locationtype')
simple('operatingsystems', OperatingSystem, 'osname')
# Per-plugin type tables exist only where that plugin is installed, so they
# are loaded through the ORM registry rather than imported directly - a lean
# build without printers must not fail here.
plugin_types = [
('machinetypes', 'machinetype'),
('computertypes', 'computertype'),
('printertypes', 'printertype'),
('networkdevicetypes', 'networkdevicetype'),
]
by_table = {m.class_.__tablename__: m.class_ for m in db.Model.registry.mappers}
for key, column in plugin_types:
model = by_table.get(key)
if model is None:
continue
simple(key, model, column)
# Vendors before models, since a model resolves its vendor by name.
count = 0
for record in data.get('vendors') or []:
name = record.get('vendor')
if not name or Vendor.query.filter_by(vendor=name).first():
continue
db.session.add(Vendor(**{k: v for k, v in record.items() if hasattr(Vendor, k)}))
count += 1
note('vendors', count)
db.session.flush()
vendor_ids = {v.vendor: v.vendorid for v in Vendor.query.all()}
modeltype_ids = {m.modeltype: m.modeltypeid for m in ModelType.query.all()}
count = 0
skipped_vendor = 0
for record in data.get('models') or []:
modelnumber = record.get('modelnumber')
if not modelnumber:
continue
vendorid = vendor_ids.get(record.get('vendor'))
# The catalog's unique key is model number PLUS vendor, so the same
# number from two makers stays two records.
if Model.query.filter_by(modelnumber=modelnumber, vendorid=vendorid).first():
continue
if record.get('vendor') and vendorid is None:
skipped_vendor += 1
continue
db.session.add(Model(
modelnumber=modelnumber,
vendorid=vendorid,
modeltypeid=modeltype_ids.get(record.get('modeltype')),
description=record.get('description'),
documentationurl=record.get('documentationurl'),
imageurl=record.get('imageurl'),
))
count += 1
note('models', count)
# Small plugin vocabularies, loaded through the registry so a lean build
# missing that plugin skips them instead of failing to import.
for key, model_table, column in (('measuringtooltypes', 'measuringtooltypes', 'name'),
('notificationtypes', 'notificationtypes', 'typename'),
('accessprotocols', 'accessprotocols', 'name')):
model = by_table.get(model_table)
if model is not None:
simple(key, model, column)
# Printer supplies. Resolved against the models loaded above, by the model's
# natural key - part numbers are useless attached to the wrong printer.
supply_model = by_table.get('modelsupplies')
if supply_model is not None and (data.get('modelsupplies') or []):
db.session.flush()
model_key = {}
for m in Model.query.all():
model_key[(m.modelnumber, m.vendorid)] = m.modelnumberid
vendor_ids = {v.vendor: v.vendorid for v in Vendor.query.all()}
count = 0
orphaned = 0
for record in data['modelsupplies']:
partnumber = record.get('partnumber')
modelnumber = record.get('modelnumber')
if not partnumber or not modelnumber:
continue
modelnumberid = model_key.get((modelnumber, vendor_ids.get(record.get('vendor'))))
if modelnumberid is None:
orphaned += 1
continue
if db.session.query(supply_model).filter_by(
modelnumberid=modelnumberid, partnumber=partnumber).first():
continue
db.session.add(supply_model(
modelnumberid=modelnumberid,
supplytype=record.get('supplytype') or 'toner',
color=record.get('color') or 'none',
capacitytier=record.get('capacitytier') or 'standard',
partnumber=partnumber,
marketingname=record.get('marketingname'),
pageyield=record.get('pageyield'),
notes=record.get('notes'),
))
count += 1
note('modelsupplies', count)
if orphaned:
click.echo(click.style(' %d supply record(s) skipped: their model is not in this catalog'
% orphaned, fg='yellow'))
if dry_run:
db.session.rollback()
click.echo(click.style('DRY RUN - nothing written.', fg='yellow'))
else:
db.session.commit()
if not added:
click.echo(click.style('Catalog already present, nothing to add.', fg='green'))
else:
for kind in sorted(added):
click.echo(' %-22s +%d' % (kind, added[kind]))
click.echo('')
click.echo(click.style('Catalog loaded from %s' % os.path.basename(path), fg='green'))
if skipped_vendor:
click.echo(click.style(' %d model(s) skipped: their vendor is not in this catalog'
% skipped_vendor, fg='yellow'))
@seed_cli.command('reference-data')
@with_appcontext
def seed_reference_data():

2779
shopdb/data/catalog.json Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,81 @@
"""Store an empty optional unique field as NULL, not as an empty string.
A column that is `unique=True` and nullable accepts any number of NULLs, which
is what makes "optional but unique" work at all. It accepts exactly ONE empty
string. So the first record saved with a blank code succeeds and the second
fails with "Duplicate entry '' for key", surfacing as a 500 from an endpoint
that did nothing wrong.
That is not a business-unit problem. A dozen columns across core and the plugins
are unique and nullable - asset numbers, host names, item codes, subnet names -
and every one of them is a blank web form away from the same collision. Fixing
them one endpoint at a time would leave the next one to be discovered by a user.
So it is fixed once, at the mapper: before any insert or update, a string column
that is unique and nullable and has been set to '' or whitespace is stored as
NULL instead. An empty string carries no information in a unique column - there
is no case where two records both meaning "blank" should collide - so nothing is
lost by normalising it.
Registered from create_app. Applies to every model, including plugin models
loaded later, because it hooks the shared Mapper class rather than a list of
models known at import time.
"""
from sqlalchemy import event, String, inspect
from sqlalchemy.orm import Session
_CACHE = {}
def _blankable_columns(mapper):
"""Unique, nullable, string columns on this mapper - worked out once each."""
cached = _CACHE.get(mapper)
if cached is not None:
return cached
columns = []
for prop in mapper.column_attrs:
for column in prop.columns:
if not column.unique or not column.nullable:
continue
# Only text. A unique nullable integer cannot be handed '' by a form
# without failing type coercion long before it reaches here.
if not isinstance(column.type, String):
continue
columns.append(prop.key)
break
_CACHE[mapper] = columns
return columns
def _normalise_instance(obj):
try:
mapper = inspect(obj).mapper
except Exception:
return
for key in _blankable_columns(mapper):
value = getattr(obj, key, None)
if isinstance(value, str) and not value.strip():
setattr(obj, key, None)
def _before_flush(session, _flush_context, _instances):
# Both new and modified: editing a record to clear its code has to become
# NULL for the same reason creating one with a blank code does.
for obj in list(session.new) + list(session.dirty):
_normalise_instance(obj)
def register_blank_unique_normaliser():
"""Hook every session flush, for every model including plugins'.
Listening on Session rather than on individual mappers means models imported
later - which is every plugin model - are covered without registration, and
it avoids depending on mapper-level event semantics that differ across
SQLAlchemy versions.
"""
if getattr(register_blank_unique_normaliser, '_installed', False):
return
event.listen(Session, 'before_flush', _before_flush)
register_blank_unique_normaliser._installed = True