ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge origin/main into wilm: dashboard/fleet console, vuln upgrades, colours

Browse Source
This commit is contained in:
benjamin.baker
2026-08-30 14:41:56 -04:00
parent c3875b0eec 8f4fe74dcb
commit d6cd0c91b3
53 changed files with 3671 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

379
scripts/scan_sbom.py Normal file
View File

@@ -0,0 +1,379 @@
#!/usr/bin/env python3
"""Check the SBOM against OSV, and say what actually needs doing.
python3 scripts/scan_sbom.py scan, report, exit 0
python3 scripts/scan_sbom.py --gate exit 1 on a shipped finding
python3 scripts/scan_sbom.py --json out.json machine-readable too
python3 scripts/scan_sbom.py --sbom path.json scan an existing SBOM
generate_sbom.py already answers "what are we running": 345 components, each
with a purl and a hash. Nothing consumed that file, so "are we exposed" was
still a person going and looking. This closes that.
WHY OSV, AND NOT A SCANNER
OSV.dev takes a purl and answers with the advisories that apply. No account, no
API key, no vendor agent, no database to keep current on an air-gapped box. The
SBOM is already the input it wants, so this is a lookup rather than another
scan of the tree - which also means it cannot disagree with what shipped.
SHIPPED VERSUS DEV-ONLY IS THE WHOLE POINT
Two thirds of the components here are build tooling that never reaches a
server, and the SBOM already marks them scope 'excluded'. A scanner that fails
a release over a vite transitive dependency teaches everybody to ignore it
within a month, so dev-only findings are REPORTED and never gate. Only what
ships can stop a build.
SEVERITY IS TAKEN, NOT COMPUTED
An OSV entry carries whatever the upstream database recorded. Where that is a
plain word (GHSA gives LOW/MODERATE/HIGH/CRITICAL) it is used as-is. Where
there is only a CVSS vector, this reports UNKNOWN rather than parsing the
vector into a score it might get wrong. UNKNOWN is shown and never gates: a
guessed score is worse than an honest gap, because somebody would act on it.
TRIAGE IS A FILE, NOT A MEMORY
scripts/vulnerability-allowlist.json records advisories reviewed and accepted,
each with a reason and a date. That is what turns a recurring report into a
record: "we looked at this on the 5th, here is why it does not apply" outlives
the person who worked it out.
"""
import argparse
import json
import pathlib
import sys
import urllib.error
import urllib.request
REPO = pathlib.Path(__file__).resolve().parent.parent
DEFAULT_SBOM = REPO / 'sbom.cdx.json'
ALLOWLIST = REPO / 'scripts' / 'vulnerability-allowlist.json'
OSV_BATCH = 'https://api.osv.dev/v1/querybatch'
OSV_VULN = 'https://api.osv.dev/v1/vulns/'
# Order matters: the gate compares by index, so adding a level means deciding
# where it sits rather than hoping a string comparison does the right thing.
SEVERITY_ORDER = ['UNKNOWN', 'LOW', 'MODERATE', 'MEDIUM', 'HIGH', 'CRITICAL']
GATE_AT = 'HIGH'
# OSV batches, but not without limit, and a 345-component POST is rude.
BATCH_SIZE = 100
TIMEOUT = 40
def post(url, payload):
body = json.dumps(payload).encode('utf8')
request = urllib.request.Request(
url, data=body, headers={'Content-Type': 'application/json'})
with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
return json.loads(response.read().decode('utf8'))
def get(url):
with urllib.request.urlopen(url, timeout=TIMEOUT) as response:
return json.loads(response.read().decode('utf8'))
def load_components(path):
"""[(purl, name, version, shipped)] from a CycloneDX document."""
document = json.loads(path.read_text(encoding='utf8'))
out = []
for component in document.get('components', []):
purl = component.get('purl')
if not purl:
continue
out.append((
purl,
component.get('name', purl),
component.get('version', ''),
component.get('scope', 'required') != 'excluded',
))
return out
def severity_of(entry):
"""The severity the upstream database recorded, or UNKNOWN.
Deliberately not derived from a CVSS vector. See the module docstring.
"""
specific = entry.get('database_specific') or {}
word = str(specific.get('severity') or '').strip().upper()
if word in SEVERITY_ORDER:
return word
return 'UNKNOWN'
# OSV names ecosystems in its own vocabulary; a purl uses another. Only the two
# this SBOM produces are mapped, because a wrong guess here silently attributes
# some other package's fix version to this component.
PURL_ECOSYSTEM = {'npm': 'npm', 'pypi': 'PyPI'}
def parse_purl(purl):
"""('npm', 'axios') from 'pkg:npm/axios@1.13.2'. Scopes survive."""
body = purl[4:] if purl.startswith('pkg:') else purl
body = body.split('?')[0].split('#')[0]
kind, _, rest = body.partition('/')
name = rest.rsplit('@', 1)[0] if '@' in rest[1:] else rest
from urllib.parse import unquote
return PURL_ECOSYSTEM.get(kind.lower(), kind), unquote(name)
def fixed_versions(entry, purl):
"""Versions upstream says carry the fix, for THIS package only.
Reported as a list rather than a single 'upgrade to X' because an advisory
that was patched on several release branches has several right answers, and
picking one for the reader would be guessing at their constraints.
"""
ecosystem, name = parse_purl(purl)
out = []
for affected in entry.get('affected', []):
package = affected.get('package') or {}
if package.get('ecosystem') != ecosystem:
continue
if str(package.get('name', '')).lower() != name.lower():
continue
for range_ in affected.get('ranges', []):
for event in range_.get('events', []):
version = event.get('fixed')
if version and version not in out:
out.append(version)
return out
def at_or_above(severity, floor):
try:
return SEVERITY_ORDER.index(severity) >= SEVERITY_ORDER.index(floor)
except ValueError:
return False
def load_allowlist():
if not ALLOWLIST.is_file():
return {}
data = json.loads(ALLOWLIST.read_text(encoding='utf8'))
return {entry['id']: entry for entry in data.get('accepted', [])}
def query(components):
"""{purl: [vuln ids]} from OSV, in batches."""
found = {}
for start in range(0, len(components), BATCH_SIZE):
chunk = components[start:start + BATCH_SIZE]
payload = {'queries': [{'package': {'purl': purl}}
for purl, _n, _v, _s in chunk]}
result = post(OSV_BATCH, payload)
for (purl, _n, _v, _s), row in zip(chunk, result.get('results', [])):
ids = [v['id'] for v in (row.get('vulns') or [])]
if ids:
found[purl] = ids
return found
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sbom', type=pathlib.Path, default=None)
parser.add_argument('--json', type=pathlib.Path, default=None,
help='also write the findings as JSON')
parser.add_argument('--gate', action='store_true',
help='exit 1 when a SHIPPED component has a finding at '
'or above %s that is not allowlisted' % GATE_AT)
args = parser.parse_args()
sbom = args.sbom or DEFAULT_SBOM
if not sbom.is_file():
# Generate rather than fail: the input is derived from files in the
# repo, so there is no reason to make somebody run two commands.
sys.stderr.write('No SBOM at %s - generating one.\n' % sbom)
sbom = REPO / 'sbom.cdx.json'
import subprocess
subprocess.run([sys.executable, str(REPO / 'scripts' / 'generate_sbom.py'),
str(REPO), '-o', str(sbom)], check=True)
components = load_components(sbom)
shipped = {purl: ship for purl, _n, _v, ship in components}
names = {purl: '%s %s' % (n, v) for purl, n, v, _s in components}
versions = {purl: v for purl, _n, v, _s in components}
print('Scanning %d components (%d shipped) against OSV'
% (len(components), sum(1 for _p, _n, _v, s in components if s)))
try:
hits = query(components)
except (urllib.error.URLError, TimeoutError) as error:
sys.stderr.write(
'Could not reach OSV: %s\n'
'This needs outbound HTTPS. Run it where the repository lives, not '
'on an air-gapped server.\n' % error)
return 2
allowed = load_allowlist()
findings = []
for purl, ids in sorted(hits.items()):
for vuln_id in ids:
try:
entry = get(OSV_VULN + vuln_id)
except (urllib.error.URLError, TimeoutError):
entry = {'id': vuln_id, 'summary': '(details unavailable)'}
findings.append({
'id': vuln_id,
'purl': purl,
'component': names.get(purl, purl),
'shipped': shipped.get(purl, True),
'severity': severity_of(entry),
'summary': (entry.get('summary') or '').strip(),
'version': versions.get(purl, ''),
'fixedin': fixed_versions(entry, purl),
'allowlisted': vuln_id in allowed,
'reason': (allowed.get(vuln_id) or {}).get('reason'),
})
findings.sort(key=lambda f: (not f['shipped'],
-SEVERITY_ORDER.index(f['severity']),
f['component']))
blocking = [f for f in findings
if f['shipped'] and not f['allowlisted']
and at_or_above(f['severity'], GATE_AT)]
report(findings, blocking)
if args.json:
args.json.write_text(json.dumps(
{'findings': findings, 'blocking': [f['id'] for f in blocking]},
indent=2), encoding='utf8')
print('\nWrote %s' % args.json)
if args.gate and blocking:
return 1
return 0
def version_key(version):
"""Sort dotted versions numerically, so 1.10 beats 1.9.
A prerelease sorts BELOW the release it precedes, so 2.0.0 beats
2.0.0-rc1. Treating '-rc1' as another dotted segment gets this backwards -
the longer key wins a list comparison - and the rollup would then send
somebody to a release candidate. The release marker is the last element for
exactly that reason.
"""
text = str(version)
base, _, pre = text.partition('-')
parts = []
for chunk in base.split('.'):
parts.append((1, int(chunk)) if chunk.isdigit() else (0, 0))
# 1 = a real release, 0 = a prerelease of it.
parts.append((0,) if pre else (1,))
return parts
def minimal_fix(fixedin, current):
"""The SMALLEST published fix above the version in use, or None.
An advisory patched on several release branches lists a fix per branch:
nanoid's is fixed in both 3.3.18 and 5.1.16. Taking the highest recommends
a major upgrade when a patch on the current branch would do - and where the
package is transitive, that major may not even be reachable, because the
parent's semver range forbids it. The smallest fix above the current
version is the one that actually clears the advisory.
Fixes at or below the current version belong to older branches and are
ignored; recommending one would be a downgrade.
"""
above = [v for v in fixedin if version_key(v) > version_key(current)]
if not above:
return None
return min(above, key=version_key)
def upgrade_plan(findings):
"""[(component, version, count, major)] - one row per component with a fix.
Per advisory, the minimal upgrade that clears it; per component, the
highest of those, since one upgrade has to satisfy every advisory against
it.
"""
by_component = {}
for f in findings:
if f['allowlisted'] or not f['fixedin']:
continue
target = minimal_fix(f['fixedin'], f['version'])
if target is None:
continue
name = f['component']
best, count, current = by_component.get(name, (None, 0, f['version']))
if best is None or version_key(target) > version_key(best):
best = target
by_component[name] = (best, count + 1, current)
rows = []
for name, (best, count, current) in by_component.items():
# A major bump is a different job from a patch bump - it can be
# ESM-only, or drop an API - so it is labelled rather than listed
# beside patch upgrades as though the two cost the same.
rows.append((name, best, count, major_of(best) != major_of(current)))
return sorted(rows)
def major_of(version):
head = str(version).split('.')[0]
return int(head) if head.isdigit() else None
def report(findings, blocking):
if not findings:
print('\nNo advisories match any component. Nothing to do.')
return
ship = [f for f in findings if f['shipped']]
dev = [f for f in findings if not f['shipped']]
print('\n%d advisory match(es): %d against shipped components, %d against '
'build-only tooling.' % (len(findings), len(ship), len(dev)))
if ship:
print('\nSHIPPED - these reach a server')
for f in ship:
mark = 'allowed' if f['allowlisted'] else ''
fix = ('fix: ' + ', '.join(f['fixedin'][:3])) if f['fixedin'] else 'no fix published'
print(' [%-8s] %-30s %-20s %-16s %s'
% (f['severity'], f['component'][:30], f['id'], fix[:16], mark))
if f['reason']:
print(' accepted: %s' % f['reason'])
elif f['summary']:
print(' %s' % f['summary'][:92])
if dev:
print('\nBUILD ONLY - never reaches a server, never gates a release')
for f in dev:
print(' [%-8s] %-34s %s'
% (f['severity'], f['component'][:34], f['id']))
if ship:
upgrades = upgrade_plan(ship)
if upgrades:
print('\nWHAT TO UPGRADE - shipped components, smallest upgrade that clears them')
for component, version, count, major in upgrades:
note = ' MAJOR - check for breaking changes' if major else ''
print(' %-30s -> %-12s clears %-3d%s'
% (component[:30], version, count, note))
print('')
if blocking:
print('%d shipped finding(s) at or above %s and not allowlisted.'
% (len(blocking), GATE_AT))
print('Fix the dependency, or record a decision in %s with a reason '
'and a date.' % ALLOWLIST.relative_to(REPO))
else:
print('Nothing shipped is at or above %s outside the allowlist.' % GATE_AT)
if __name__ == '__main__':
sys.exit(main())

43
scripts/site_imports/wjf/PILOT-DEPLOY.md
View File

@@ -78,9 +78,36 @@ import API. It is site glue, not product code.
venv/bin/python -m scripts.site_imports.wjf.run
```
The 16 stages run in order (reference -> catalog -> assets hub -> locations ->
printers -> dependents -> relationships -> subnets -> usb -> verify). It is
idempotent - a crashed run resumes from `idmap.json`.
The 17 stages run in order (reference -> employees -> catalog -> assets hub
-> locations -> printers -> metrology -> communications -> applications ->
warranties -> notifications -> knowledgebase -> relationships ->
defaultprinters -> subnets -> usb -> verify).
**Pointing it at a different source.** The three source databases are
named logically in the loader and resolved from the environment, so a
restore under another name needs no code edit:
```
SOURCE_DB_MAIN=prodscratch \
SOURCE_DB_USB=cmmc_usb_src \
SOURCE_DB_EMPLOYEES=wjf_employees_src \
DATABASE_URL=... venv/bin/python -m scripts.site_imports.wjf.run
```
Also `SOURCE_DB_HOST`, `SOURCE_DB_PORT`, `SOURCE_DB_USER`,
`SOURCE_DB_PASSWORD`. The run prints every source it redirected, so
"it imported nothing" can be told apart from "it read the wrong database".
**Resume is per target database.** A crashed run resumes from the crosswalk
in `idmap.json`, which maps this site's legacy ids to assetids **in one
specific database**. The loader stamps that database's identity into the
file and refuses a crosswalk built against a different one, because reusing
it attaches warranties and printer assignments to whatever asset now holds
that id - wrong data, no error. If you rebuild the target, delete
`idmap.json` or pass `--idmap` a fresh path.
The exit code means something: non-zero if any endpoint errored, so a run
that logged 200 failures cannot read as a success in a deploy log.
3. Reclassify servers into network devices. The classic DB stored servers as
PCs, so the import lands them as `computer` assets. Re-point them in place:
@@ -151,8 +178,12 @@ documented skips (inactive rows, duplicate machinenumbers, LocationOnly, the
- Keep the classic app authoritative during the window. The pilot is read-mostly
for validation; do not dual-write.
- Have a few real users (IT + a floor lead) work the pilot and log gaps.
- Re-import is cheap: fix a loader mapping, drop + rebuild the pilot DB, re-run.
Nothing you do to the pilot touches classic.
- Re-import is cheap: fix a loader mapping, drop + rebuild the pilot DB,
**delete `idmap.json`**, re-run. Nothing you do to the pilot touches classic.
The crosswalk is bound to the database it was built against and the loader
refuses a stale one, but deleting it is the habit to have - a refusal costs
you a restart, and before the guard existed a stale file silently attached
rows to the wrong assets.
- Point the **collector** (GE-Enforce fleet ingest) at the pilot in parallel to
confirm live PC check-ins land (see `COLLECTOR-INTEGRATION.md`), using a
scoped service token.
@@ -163,7 +194,7 @@ When the window is clean:
1. Freeze classic writes (announce a short read-only window).
2. Take final fresh dumps; re-run the loader into a clean pilot DB so the
cutover data is current.
cutover data is current. A clean DB means a fresh `idmap.json` too.
3. Verify counts + a fast UI spot-check.
4. Repoint the production hostname/DNS (or the reverse proxy) at the pilot.
5. Retire the classic app to read-only standby (do not delete - keep it as the

34
scripts/site_imports/wjf/README.md
View File

@@ -33,16 +33,34 @@ resolved decisions, and remaining stages.
## Status
Complete - all 15 stages built and verified end-to-end against a fresh scratch
target (zero endpoint errors): `reference`, `employees`, `catalog`, `assets`
(the hub + machineid->assetid crosswalk), `locations`, `printers`,
All 17 stages built: `reference`, `employees`, `catalog`, `assets` (the hub +
machineid->assetid crosswalk), `locations`, `printers`, `metrology`,
`communications`, `applications` (+ installs), `warranties`, `notifications`,
`knowledgebase`, `relationships`, `subnets`, `usb`, `verify`.
`knowledgebase`, `relationships`, `defaultprinters`, `subnets`, `usb`, `verify`.
Last full run: 983 assets (computer 663, machine 76, network 58, measuring-tool
136, printer 50), 24 locations, 415 employees, 850 installs, 461 primary IPs,
464 warranties, 261 notifications, 341 KB, 93 relationships, 37 subnets, 18 USB
devices + 232 events. UI spot-check passed (computer list + shop-floor map).
**The counts below are from the 2026-07-13 run and are STALE.** Two routing
changes landed after them - the `isactive=1` filter and the metrology routing
rework - so the asset mix has moved substantially. They are kept only as a
shape, not as a checklist: re-measure from the next full run before quoting
them anywhere, and do not verify a fresh import against them.
Last full run (2026-07-13, superseded): 983 assets (computer 663, machine 76,
network 58, measuring-tool 136, printer 50), 24 locations, 415 employees, 850
installs, 461 primary IPs, 464 warranties, 261 notifications, 341 KB, 93
relationships, 37 subnets, 18 USB devices + 232 events.
Known gaps, none of them blocking a scratch run:
- `warranties`, `notifications` and `knowledgebase` POST without a duplicate
check on the API side, so resuming ACROSS one of those stages doubles its
rows. Resuming at or after them is safe. Re-run into a clean target when in
doubt.
- There is no `--dry-run` and no guard on the target: the harness mints an
admin PAT and sets `setup_complete` in whatever `DATABASE_URL` points at.
Check the target before running.
- The `verify` stage counts target `assets` against source `machines`, which is
apples to oranges - the target also holds printers and synthesized metrology
tools.
Known follow-ups: general asset `locationid` is null (no source column outside
printers); the ~146 duplicate machinenumbers are first-wins-skipped by decision.

137
scripts/site_imports/wjf/harness.py
View File

@@ -14,6 +14,7 @@ crashed run resumes and later stages resolve foreign keys.
"""
import json
import uuid
import os
import pymysql
@@ -30,26 +31,63 @@ def _source_password():
with open(os.path.abspath(env)) as handle:
for line in handle:
if line.startswith('DATABASE_URL='):
# Not every server runs as root, and 'root:' then simply is not
# there: split()[1] raised IndexError, which read as a crash
# rather than the clear message two lines down.
if 'root:' not in line:
continue
after = line.split('root:', 1)[1]
return after.split('@', 1)[0]
raise RuntimeError('could not resolve source DB password')
class Source:
"""Read-only access to the three scratch source databases."""
"""Read-only access to the three source databases.
def __init__(self, host='127.0.0.1', port=3306, user='root', password=None):
The stage functions name a source by its LOGICAL name - 'shopdb_src',
'cmmc_usb_src', 'wjf_employees_src' - and this class resolves each to a
real database. That indirection is the point: the names were hardcoded at
29 call sites, so pointing the loader at a differently-named copy (the
prodscratch restore, or a site's own dump) meant editing run.py, which is
not something anybody should do to aim a loader. The logical names stay in
the stages where they read as documentation; the mapping lives here.
Everything is environment-overridable, defaulting to the scratch layout:
SOURCE_DB_MAIN default shopdb_src
SOURCE_DB_USB default cmmc_usb_src
SOURCE_DB_EMPLOYEES default wjf_employees_src
SOURCE_DB_HOST/PORT/USER/PASSWORD
"""
# logical name -> (env var, default real name)
ALIASES = {
'shopdb_src': ('SOURCE_DB_MAIN', 'shopdb_src'),
'cmmc_usb_src': ('SOURCE_DB_USB', 'cmmc_usb_src'),
'wjf_employees_src': ('SOURCE_DB_EMPLOYEES', 'wjf_employees_src'),
}
def __init__(self, host=None, port=None, user=None, password=None):
self.password = password or _source_password()
self.host, self.port, self.user = host, port, user
self.host = host or os.environ.get('SOURCE_DB_HOST', '127.0.0.1')
self.port = int(port or os.environ.get('SOURCE_DB_PORT', 3306))
self.user = user or os.environ.get('SOURCE_DB_USER', 'root')
self._conns = {}
@classmethod
def resolve(cls, name):
"""The real database behind a logical source name."""
env, default = cls.ALIASES.get(name, (None, name))
return os.environ.get(env, default) if env else name
def db(self, name):
if name not in self._conns:
self._conns[name] = pymysql.connect(
real = self.resolve(name)
if real not in self._conns:
self._conns[real] = pymysql.connect(
host=self.host, port=self.port, user=self.user,
password=self.password, database=name, charset='utf8mb4',
password=self.password, database=real, charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
return self._conns[name]
return self._conns[real]
def rows(self, dbname, sql, args=None):
with self.db(dbname).cursor() as cur:
@@ -57,15 +95,62 @@ class Source:
return cur.fetchall()
class IdMap:
"""Persisted legacy-id -> new-id crosswalks, one namespace per entity."""
class StaleIdMapError(RuntimeError):
"""The crosswalk on disk was built against a different target database."""
def __init__(self, path):
class IdMap:
"""Persisted legacy-id -> new-id crosswalks, one namespace per entity.
BOUND TO ONE TARGET DATABASE. The crosswalk holds this site's legacy ids
mapped to assetids in a specific database, and those assetids mean nothing
anywhere else. Two ways that used to go wrong, both silent:
- Delete idmap.json and re-run: every create 409s, nothing resolves into
the crosswalk, and warranties, installs, relationships and printer
assignments all quietly import as ~0 rows.
- Keep idmap.json across a drop-and-rebuild of the target: the legacy ids
now point at assetids belonging to the OLD database, so warranties and
printer assignments attach to whatever asset happens to hold that id
now. Wrong data, no error, and the runbook's two "drop and rebuild"
steps never said to delete the file.
The bind is a marker row in the target's own settings table, so it dies
with the data: a rebuild produces a new marker, and the mismatch is caught
before a single row is written. The database NAME is not enough on its own,
because a rebuild keeps it.
"""
MARKER_KEY = 'siteimport_targetinstance'
def __init__(self, path, target=None):
self.path = path
self.target = target
self._data = {}
if os.path.exists(path):
with open(path) as handle:
self._data = json.load(handle)
self._check_target()
def _check_target(self):
if not self.target:
return
seen = self._data.get('_target')
if seen is None:
# Written before this check existed. Adopt it rather than refuse:
# refusing would strand a crosswalk that is probably fine, and the
# next run is bound properly.
self._data['_target'] = self.target
self.save()
return
if seen != self.target:
raise StaleIdMapError(
'%s was built against target %s, but this run targets %s.\n'
'Those assetids belong to a different database and using them '
'would attach imported rows to the wrong assets.\n'
'Delete the file to start a fresh crosswalk, or point '
'--idmap at the one for this target.'
% (self.path, seen, self.target))
def put(self, namespace, legacy, new):
self._data.setdefault(namespace, {})[str(legacy)] = new
@@ -78,7 +163,8 @@ class IdMap:
return str(legacy) in self._data.get(namespace, {})
def count(self, namespace):
return len(self._data.get(namespace, {}))
entry = self._data.get(namespace, {})
return len(entry) if isinstance(entry, dict) else 0
def save(self):
with open(self.path, 'w') as handle:
@@ -97,11 +183,38 @@ class Harness:
self._ensure_asset_types()
self.secret = self._mint_admin_pat()
default = os.path.join(os.path.dirname(__file__), 'idmap.json')
self.ids = IdMap(idmap_path or default)
self.ids = IdMap(idmap_path or default, target=self._target_instance())
self.source = Source()
self.errors = []
self._defaultlevelid = None
def _target_instance(self):
"""A stable id for THIS target database, created once and stored in it.
Kept in the target's own settings table so it dies when the data does.
A drop-and-rebuild yields a new id, which is what lets IdMap catch a
crosswalk carried over from the previous database - the failure that
silently attaches imported rows to whatever asset now holds that id.
The database name alone would not do: a rebuild keeps the name.
"""
from shopdb.core.models import Setting
existing = Setting.query.filter_by(key=IdMap.MARKER_KEY).first()
if existing and existing.value:
return existing.value
marker = uuid.uuid4().hex
if existing:
existing.value = marker
else:
db.session.add(Setting(
key=IdMap.MARKER_KEY, value=marker, valuetype='string',
category='import',
description='Identifies this database to the site-import '
'crosswalk. Created by the loader; do not copy it '
'between databases.'))
db.session.commit()
return marker
@property
def defaultlevelid(self):
"""The level imported map positions belong to (ADR-017).

138
scripts/site_imports/wjf/run.py
View File

@@ -9,16 +9,19 @@ Stages are idempotent (re-running resolves existing rows by natural key), so a
crashed run resumes. See scripts/site_imports/wjf/harness.py for the contract
this drives, and scratchpad/IMPORT-PLAN.md for the full mapping + decisions.
Implemented: reference (vendors, businessunits, operatingsystems), employees.
TODO stages (assets hub + dependents + network + usb) are stubbed - they need
the machineid->assetid crosswalk this harness persists.
All 17 stages are implemented; STAGES at the bottom of this file is the list
and main()'s --stages default is the order they must run in. The assets hub
comes before everything that references an asset, because it is what builds the
machineid->assetid crosswalk the harness persists.
"""
import argparse
import sys
from urllib.parse import quote
import pymysql
from .harness import Harness
from .harness import Harness, Source, StaleIdMapError
def _id_of(row, idfield):
@@ -33,6 +36,76 @@ def _id_of(row, idfield):
return None
# The server clamps any page size to MAX_PAGE_SIZE (shopdb/config.py), which
# is 100. Asking for more is not an error and not a warning - it just silently
# returns 100 rows, which is why the 409-resolve below pages instead.
PAGE_SIZE = 100
# A resolve that walks more pages than this is a bug in the caller, not a big
# table. Bounded so a paging mistake cannot spin forever against a live server.
MAX_RESOLVE_PAGES = 200
def _rows_of(payload):
"""The list out of a list-endpoint response, whichever shape it took."""
if isinstance(payload, list):
return payload
if isinstance(payload, dict):
items = payload.get('items')
if isinstance(items, list):
return items
return []
def _resolve_existing(h, lookup, unique_field, wanted, idfield):
"""Find an existing row by its natural key. Returns its id, or None.
This used to ask for ?per_page=10000 and read the single response. The
server clamps per_page to 100, so it only ever searched the first 100 rows
and returned None for everything after that. The failure was silent and
downstream: _upsert returned None, the id never reached the crosswalk, and
the run finished "successfully" having quietly dropped a model's link on
every machine that used it. The target already holds more than 100 models
and more than 100 applications, so this fired on any re-run or resume - the
exact case the loader documents as safe.
Two passes. Some endpoints expose an exact natural-key filter for this
(?modelnumber= on models, ?appname= on applications, ?vendor= on vendors);
an endpoint without one ignores the unknown query parameter and returns its
normal first page, so the paged walk that follows is what finds the row.
"""
wanted_key = str(wanted).strip().lower()
separator = '&' if '?' in lookup else '?'
def match(rows):
for row in rows:
if str(row.get(unique_field, '')).strip().lower() == wanted_key:
return _id_of(row, idfield)
return None
# Pass 1: the exact filter, where the endpoint has one.
exact = '%s%s%s=%s&per_page=%d' % (
lookup, separator, unique_field, quote(str(wanted)), PAGE_SIZE)
_, data = h.get(exact)
found = match(_rows_of(data))
if found is not None:
return found
# Pass 2: every page, in order.
for page in range(1, MAX_RESOLVE_PAGES + 1):
_, data = h.get('%s%sper_page=%d&page=%d'
% (lookup, separator, PAGE_SIZE, page))
rows = _rows_of(data)
if not rows:
return None
found = match(rows)
if found is not None:
return found
if len(rows) < PAGE_SIZE:
return None
return None
def _upsert(h, path, payload, unique_field, idfield, list_path=None):
"""POST payload; on 409 resolve the existing row by its unique field.
Returns the new/existing id, or None."""
@@ -40,14 +113,8 @@ def _upsert(h, path, payload, unique_field, idfield, list_path=None):
if status in (200, 201):
return _id_of(data, idfield)
if status == 409:
lookup = list_path or path
lookup += ('&' if '?' in lookup else '?') + 'per_page=10000'
_, rows = h.get(lookup)
items = rows if isinstance(rows, list) else rows.get('items', rows)
for row in (items or []):
if str(row.get(unique_field, '')).strip().lower() == \
str(payload[unique_field]).strip().lower():
return _id_of(row, idfield)
return _resolve_existing(h, list_path or path, unique_field,
payload[unique_field], idfield)
return None
@@ -418,10 +485,22 @@ def stage_metrology(h):
'businessunitid': h.ids.get('businessunit', m['businessunitid']),
'mapx': m['mapleft'], 'mapy': m['maptop'],
'levelid': h.defaultlevelid})
if status not in (200, 201):
if status in (200, 201):
tool_assetid = _id_of(data, 'assetid')
made += 1
elif status == 409:
# Already imported by an earlier run. Resolve it rather than
# skipping: `continue` here left the crosswalk without this tool,
# so the Controls edge below was never made and a resumed run
# produced metrology PCs driving nothing. stage_printers already
# had this branch; this stage was written later and missed it.
tool_assetid = _resolve_existing(
h, '/api/measuringtools', 'assetnumber',
f"MT-{m['machineid']}", 'assetid')
if not tool_assetid:
continue
else:
continue
tool_assetid = _id_of(data, 'assetid')
made += 1
if tool_assetid and controls_id:
h.ids.put('metrologytool', m['machineid'], tool_assetid)
st, _ = h.post('/api/assets/relationships', {
@@ -906,10 +985,27 @@ def main():
'metrology,communications,applications,warranties,notifications,'
'knowledgebase,relationships,defaultprinters,subnets,usb,verify',
help='comma list of stages to run')
parser.add_argument(
'--idmap', default=None,
help='path to the legacy-id crosswalk (default: idmap.json beside '
'this script). One per target database - a crosswalk from '
'another database maps to the wrong assets.')
args = parser.parse_args()
h = Harness()
print(f'== WJ import loader (target: {h.app.config["SQLALCHEMY_DATABASE_URI"].split("/")[-1]}) ==')
try:
h = Harness(idmap_path=args.idmap)
except StaleIdMapError as error:
print('REFUSED: %s' % error)
return 1
target = h.app.config['SQLALCHEMY_DATABASE_URI'].split('/')[-1]
print(f'== WJ import loader (target: {target}) ==')
# Say which sources this run reads. They are overridable now, so "it
# imported nothing" is usually "it read the wrong database".
for logical in ('shopdb_src', 'cmmc_usb_src', 'wjf_employees_src'):
real = Source.resolve(logical)
if real != logical:
print(f' source {logical} -> {real}')
try:
for name in args.stages.split(','):
name = name.strip()
@@ -922,9 +1018,15 @@ def main():
print(f'\n {len(h.errors)} endpoint error(s):')
for path, code, _payload, data in h.errors[:10]:
print(f' {code} {path}: {data}')
# An endpoint error is not a crash, but it did leave rows unimported,
# and a run that exits 0 after 200 of them looks like a success in a
# deploy log.
if h.errors:
return 1
finally:
h.close()
return 0
if __name__ == '__main__':
main()
sys.exit(main())

23
scripts/vulnerability-allowlist.json Normal file
View File

@@ -0,0 +1,23 @@
{
"_comment": [
"Advisories reviewed and accepted, with the reason and the date.",
"This is a RECORD, not a mute button: an entry says somebody looked at",
"the advisory and worked out why it does not apply here. Anyone can",
"re-read the reason and disagree.",
"",
"An entry needs an id, a reason that names the precondition that is",
"absent, who accepted it, and when. 'reviewuntil' is when the reasoning",
"should be checked again - a reason stops being true when the code that",
"made it true changes, and 'invalidatedby' should say what that change",
"would be.",
"",
"Currently EMPTY, which is the state to aim for. The one entry this file",
"has held so far - a PyJWT algorithm-confusion advisory that needed a",
"configuration this application does not use - was removed by upgrading",
"PyJWT rather than by keeping the exception.",
"",
"Do NOT add an entry to make a build pass. Upgrading the dependency is",
"almost always cheaper than defending a decision to skip it."
],
"accepted": []
}