Bring the wilm branch into main: endpoints plugin, agent, policies, CSV import
Ports the corporate-GitHub wilm branch onto main, keeping the two branches
separate. Thirteen upstream commits arrive with their authorship intact; the
review fixes and the portability work below are on top.
What arrives:
endpoints plugin - a ShopDB agent reporting endpoint state, telemetry,
services and interfaces, with a command channel the agent
long-polls. Ingest stays on the generic collector route
(ADR-006); the agent key is a UI-managed single-scope
token whose revocation is an instant kill switch.
computers policies - Configuration and Business Policies (RMM-style), scope
trees validated at write time, actions checked against
the endpoints command allowlist. Feature-gated: every
route 404s while the setting is off, so enabling it can
never half-expose the surface.
asset CSV import - templates, a collector script carrying the site path, and
a validate-then-commit import. A plugin can own a virtual
column plus a post-write row hook, which is how a PC's
machine number becomes a controls link without a
hand-built relationships sheet.
agent/ - the Windows agent and its installer.
Contract 0.21.0 -> 0.24.0, additive throughout: register_import_hook (0.22),
mint/list/revoke_service_token (0.23), collector_authorized (0.24). The stability
table had no rows for any of them and now does.
MySQL 5.6 portability. The branch stored three policy columns as sa.JSON, the
only JSON columns in the product, and 5.6 has no JSON type: the migration failed
with a bare syntax error near the word JSON. They are TEXT now, through a
JSONText type decorator, so all seventeen call sites still read and write dicts.
The convention elsewhere is already TEXT (guides.blocksjson,
geenforce.manifestjson).
Review fixes carried on top:
- six policy GETs were jwt_required(optional=True), which authenticates a
caller without checking anything; they require computers.view now, and six
redundant jwt_required decorators are gone
- the agent installer took a default BaseUrl and CollectorKey, so a mistyped
run silently pointed an endpoint at the wrong instance; both are mandatory
and a non-https BaseUrl throws unless -AllowInsecureHttp is passed
- a committing CSV import wrote no audit row
Documentation: the 34 new routes are described in docs/api-inventory.json, which
is hand-maintained on purpose, and the generated spec now covers 331 paths and
477 operations across 22 surfaces. DEVELOPMENT-SETUP's 1071 troubleshooting row
said "use MySQL 8 for dev" and now gives the actual fix, including the trap that
5.6 downgrades ROW_FORMAT=DYNAMIC to Compact with a warning instead of an error.
Full suite 2123 passed, vitest 260 passed, all seventeen plugin chains migrate
clean on MySQL 5.6 and the naming, docs, parity and read-surface gates are green.
This commit is contained in:
@@ -86,7 +86,15 @@ EXEMPT_ENDPOINTS = {'knowledgebase.track_click', 'users.update_user',
|
||||
# GET urls and a preview must not consume a one-view link.
|
||||
# CREATING one is permission-gated and IS swept.
|
||||
# Justification in plugins/tools/README.md.
|
||||
'tools.redeem_secret'}
|
||||
'tools.redeem_secret',
|
||||
# Agent-facing command result callback. Authenticated by the
|
||||
# endpoints collector API key (same shape as the exempt
|
||||
# collector blueprint), not JWT+role: the endpoint agent has
|
||||
# no user account, the collector key is its credential and
|
||||
# kill switch. Enqueue (WebUI -> operator) IS permission-gated
|
||||
# on endpoints.command and IS swept. Contract in
|
||||
# plugins/endpoints/AGENT-CONTRACT.md section 7.
|
||||
'endpoints.report_command_result'}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -73,6 +73,16 @@ PUBLIC_ENDPOINTS = {
|
||||
# The unguessable token IS the authorisation, and the recipient of a secure
|
||||
# send has no account here.
|
||||
'tools.peek_secret',
|
||||
|
||||
# The endpoint agent long-polls for its next command. Authenticated inside
|
||||
# the handler by the endpoints collector API key - the same shape as the
|
||||
# collector blueprint - because a bay has no user session. The credential is
|
||||
# also the channel's kill switch. Note which half is public: CLAIMING a
|
||||
# queued command is agent-facing, while ENQUEUEING one is the actual
|
||||
# authority and stays gated on endpoints.command, audited with the
|
||||
# requesting user. Same reasoning as endpoints.report_command_result in
|
||||
# tests/test_core/test_authz.py.
|
||||
'endpoints.claim_next_command',
|
||||
}
|
||||
|
||||
# Reads that need a LOGIN but no named permission, because the handler decides
|
||||
|
||||
@@ -12,6 +12,7 @@ from shopdb.core.services.csvimport import (
|
||||
IMPORTABLE, ImportError_, Resolver, generate_template, import_csv,
|
||||
importable_columns, required_columns,
|
||||
)
|
||||
from shopdb.core.services.csvimport import digit_runs, normalize_label
|
||||
from shopdb.extensions import db as _db
|
||||
|
||||
|
||||
@@ -173,14 +174,17 @@ def test_generated_templates_only_contain_real_columns(db):
|
||||
|
||||
A hand-written template set had invented columns on 7 of 11 tables and named
|
||||
a table that does not exist. Generating from the schema is what makes that
|
||||
impossible - so assert the generator actually does it.
|
||||
impossible - so assert the generator actually does it. Plugin-registered
|
||||
virtual columns (machineassetnumber) are legitimate, not invented, so they
|
||||
are allowed alongside the table's real columns.
|
||||
"""
|
||||
from shopdb.core.services.csvimport import virtual_columns_for
|
||||
for table in IMPORTABLE:
|
||||
text = generate_template(table)
|
||||
header = [line for line in text.splitlines() if line and not line.startswith('#')][0]
|
||||
real = {c.name for c in importable_columns(table)}
|
||||
allowed = {c.name for c in importable_columns(table)} | virtual_columns_for(table)
|
||||
for column in header.split(','):
|
||||
assert column in real, '%s.csv offers a column that does not exist: %s' % (table, column)
|
||||
assert column in allowed, '%s.csv offers a column that does not exist: %s' % (table, column)
|
||||
|
||||
|
||||
def test_every_template_includes_its_required_columns(db):
|
||||
@@ -234,3 +238,100 @@ def test_a_dry_run_over_several_files_resolves_across_them(db):
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
_db.session.rollback()
|
||||
assert count('assets') == 0
|
||||
|
||||
|
||||
def _example_files():
|
||||
"""Reconstruct each template's example rows as an importable CSV: the header
|
||||
plus the live (uncommented) example data rows that follow it."""
|
||||
files = {}
|
||||
for table in IMPORTABLE:
|
||||
header = None
|
||||
rows = []
|
||||
for line in generate_template(table).splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
if line.startswith('#'):
|
||||
continue
|
||||
if header is None:
|
||||
header = line
|
||||
else:
|
||||
rows.append(line)
|
||||
files[table] = '\n'.join([header] + rows) + '\n'
|
||||
return files
|
||||
|
||||
|
||||
def test_template_examples_are_present_for_every_table(db):
|
||||
"""Each template ships at least one worked example row, so a downloaded
|
||||
set is a usable starter workbook rather than a bare header."""
|
||||
files = _example_files()
|
||||
for table in IMPORTABLE:
|
||||
rows = files[table].splitlines()[1:]
|
||||
assert rows, '%s.csv has no example row' % table
|
||||
|
||||
|
||||
def test_template_example_set_imports_cleanly(db):
|
||||
"""The whole curated example set must import as-is: every name a later file
|
||||
references is defined by an earlier one, with no self-references that the
|
||||
validation pass cannot resolve. This is the promise the workbook makes."""
|
||||
resolver = Resolver()
|
||||
files = _example_files()
|
||||
for table in IMPORTABLE:
|
||||
result = import_csv(table, files[table], resolver=resolver)
|
||||
assert result.ok, '%s.csv example failed: %s' % (
|
||||
table, [str(p) for p in result.problems])
|
||||
_db.session.rollback()
|
||||
|
||||
|
||||
# -- fuzzy reconciliation safety (review item 1.1) -----------------------------
|
||||
#
|
||||
# A high similarity score is the DANGEROUS case: 'Bay 310' and 'Bay 311' share
|
||||
# every word and score ~0.857, so a naive threshold silently merges two distinct
|
||||
# locations. Reconciliation now ships OFF (threshold 0) and, even when a site
|
||||
# turns it on, refuses any pair whose digit runs differ.
|
||||
|
||||
def test_digit_runs_extracts_number_groups():
|
||||
assert digit_runs(normalize_label('Bay 310')) == ['310']
|
||||
assert digit_runs(normalize_label('Precision 3660')) == ['3660']
|
||||
assert digit_runs(normalize_label('Dell Inc.')) == []
|
||||
|
||||
|
||||
def test_reconciliation_is_off_by_default(db):
|
||||
"""A fresh Resolver does not fuzzy-match: an unknown numbered bay is a row
|
||||
error, not a silent bind onto its neighbour."""
|
||||
resolver = Resolver()
|
||||
seed_reference(resolver)
|
||||
# 'Bay 3' exists from the reference seed; 'Bay 8' does not.
|
||||
result = import_csv(
|
||||
'assets',
|
||||
'assetnumber,assettypeid,locationid\nA-1,Machine,Bay 8\n',
|
||||
resolver=resolver)
|
||||
assert not result.ok
|
||||
assert not resolver.events
|
||||
|
||||
|
||||
def test_reconcile_refuses_when_digit_runs_differ(db):
|
||||
"""Even with fuzzy matching turned on, 'Bay 311' must not bind to the
|
||||
existing 'Bay 3': the digit runs differ, so it is a distinct location."""
|
||||
resolver = Resolver(reconcile_threshold=0.5)
|
||||
seed_reference(resolver)
|
||||
result = import_csv(
|
||||
'assets',
|
||||
'assetnumber,assettypeid,locationid\nB-1,Machine,Bay 311\n',
|
||||
resolver=resolver)
|
||||
assert not result.ok
|
||||
assert not resolver.events
|
||||
|
||||
|
||||
def test_reconcile_still_binds_normalized_exact(db):
|
||||
"""The normalized-exact tier is untouched: 'bay 3' folds to the existing
|
||||
'Bay 3' regardless of threshold, and is reported as an event."""
|
||||
resolver = Resolver(reconcile_threshold=0.9)
|
||||
seed_reference(resolver)
|
||||
result = import_csv(
|
||||
'assets',
|
||||
'assetnumber,assettypeid,locationid\nC-1,Machine,BAY 3\n',
|
||||
resolver=resolver)
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
_db.session.rollback()
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ following the guide is refused by the loader at startup. The plugin count was
|
||||
wrong in six more.
|
||||
|
||||
A number that is copied is a number that goes stale, and prose gives no signal
|
||||
about which lines are still true. docs/PROJECT-MAP.md is generated for exactly
|
||||
about which lines are still true. the PROJECT-MAP wiki page is generated for exactly
|
||||
this, so a document points at it instead of restating it.
|
||||
|
||||
This test is the rule. Where a version genuinely belongs in prose - the contract
|
||||
@@ -75,7 +75,7 @@ def test_no_document_declares_a_version_literal():
|
||||
if ASSIGNMENT.search(line):
|
||||
offenders.append('%s:%d %s' % (path.relative_to(REPO), number, line.strip()[:90]))
|
||||
assert not offenders, (
|
||||
'These documents declare a version literal. Point at docs/PROJECT-MAP.md, '
|
||||
'These documents declare a version literal. Point at the PROJECT-MAP wiki page, '
|
||||
'which is generated:\n ' + '\n '.join(offenders))
|
||||
|
||||
|
||||
@@ -111,14 +111,14 @@ def test_a_stated_plugin_count_matches_the_tree():
|
||||
% (path.relative_to(REPO), number, match.group(1), actual))
|
||||
assert not offenders, (
|
||||
'These documents count plugins by hand. The count is in '
|
||||
'docs/PROJECT-MAP.md:\n ' + '\n '.join(offenders))
|
||||
'the PROJECT-MAP wiki page:\n ' + '\n '.join(offenders))
|
||||
|
||||
|
||||
def test_the_map_itself_is_current():
|
||||
"""The pointer target has to be right, or every document pointing at it is
|
||||
wrong at one remove."""
|
||||
mapfile = DOCS / 'PROJECT-MAP.md'
|
||||
assert mapfile.is_file(), 'docs/PROJECT-MAP.md is missing; run scripts/gen_project_map.py'
|
||||
assert mapfile.is_file(), 'the PROJECT-MAP wiki page is missing; run scripts/gen_project_map.py'
|
||||
text = mapfile.read_text()
|
||||
for name, version in current_versions().items():
|
||||
assert version in text, (
|
||||
|
||||
201
tests/test_import_sources.py
Normal file
201
tests/test_import_sources.py
Normal file
@@ -0,0 +1,201 @@
|
||||
"""Foreign import-source adapters: transforming Azure and Thor exports.
|
||||
|
||||
These adapters are pure - they turn a foreign CSV into the canonical per-table
|
||||
CSVs the import engine already understands, and touch no database. The tests
|
||||
check the transform output directly, so they run without app context.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
|
||||
import pytest
|
||||
|
||||
from shopdb.core.services.importsources import (
|
||||
AzureDeviceGroupSource, ImportSourceError, ThorSource, detect_source,
|
||||
get_source,
|
||||
)
|
||||
|
||||
|
||||
def _rows(text):
|
||||
return list(csv.DictReader(io.StringIO(text)))
|
||||
|
||||
|
||||
AZURE_CSV = (
|
||||
'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n'
|
||||
'F13HSGH4,device,,,11111111-1111-1111-1111-111111111111,aaaa1111-bbbb-2222-cccc-333333333333,,False,False,False\n'
|
||||
'E3P5M0B4,device,,,22222222-2222-2222-2222-222222222222,dddd4444-eeee-5555-ffff-666666666666,,False,False,False\n'
|
||||
)
|
||||
|
||||
|
||||
def test_azure_matches_signature():
|
||||
header = ['displayName', 'objectType', 'deviceId', 'id']
|
||||
assert AzureDeviceGroupSource.matches(header)
|
||||
assert detect_source(header) is AzureDeviceGroupSource
|
||||
|
||||
|
||||
def test_azure_transform_builds_assets_and_computers():
|
||||
produced = AzureDeviceGroupSource.transform(AZURE_CSV)
|
||||
assert set(produced) == {'assets', 'computers'}
|
||||
|
||||
assets = _rows(produced['assets'])
|
||||
assert [a['assetnumber'] for a in assets] == ['F13HSGH4', 'E3P5M0B4']
|
||||
assert all(a['assettypeid'] == 'computer' for a in assets)
|
||||
assert all(a['statusid'] == 'In Use' for a in assets)
|
||||
|
||||
computers = _rows(produced['computers'])
|
||||
assert computers[0]['assetid'] == 'F13HSGH4'
|
||||
assert computers[0]['hostname'] == 'F13HSGH4'
|
||||
assert computers[0]['azuredeviceid'] == 'aaaa1111-bbbb-2222-cccc-333333333333'
|
||||
assert 'computertypeid' not in computers[0]
|
||||
|
||||
|
||||
def test_azure_transform_stamps_computertype():
|
||||
produced = AzureDeviceGroupSource.transform(
|
||||
AZURE_CSV, options={'computertype': 'Shopfloor PC'})
|
||||
computers = _rows(produced['computers'])
|
||||
assert all(c['computertypeid'] == 'Shopfloor PC' for c in computers)
|
||||
|
||||
|
||||
def test_azure_transform_carries_objectid():
|
||||
produced = AzureDeviceGroupSource.transform(AZURE_CSV)
|
||||
computers = _rows(produced['computers'])
|
||||
# The CSV 'id' column is the Entra directory object id.
|
||||
assert computers[0]['azureobjectid'] == '11111111-1111-1111-1111-111111111111'
|
||||
|
||||
|
||||
def test_azure_transform_stamps_category():
|
||||
produced = AzureDeviceGroupSource.transform(
|
||||
AZURE_CSV, options={'category': 'SFLD_DYN_Device_Group_X'})
|
||||
computers = _rows(produced['computers'])
|
||||
assert all(c['azuredevicecategory'] == 'SFLD_DYN_Device_Group_X'
|
||||
for c in computers)
|
||||
|
||||
|
||||
def test_azure_aligns_matched_hostname_to_existing_asset():
|
||||
# A case-insensitive hostname match routes the device to its real asset and
|
||||
# emits NO new asset row, so the import updates in place instead of creating
|
||||
# a duplicate. Unmatched devices still create a new asset keyed on hostname.
|
||||
produced = AzureDeviceGroupSource.transform(
|
||||
AZURE_CSV, options={'hostname_map': {'f13hsgh4': '13NSGH4'}})
|
||||
|
||||
computers = _rows(produced['computers'])
|
||||
by_host = {c['hostname']: c for c in computers}
|
||||
# Matched device points at the existing assetnumber, not its hostname.
|
||||
assert by_host['F13HSGH4']['assetid'] == '13NSGH4'
|
||||
# Unmatched device falls back to its hostname (new asset).
|
||||
assert by_host['E3P5M0B4']['assetid'] == 'E3P5M0B4'
|
||||
|
||||
# Only the unmatched device produces an asset row.
|
||||
assets = _rows(produced['assets'])
|
||||
assert [a['assetnumber'] for a in assets] == ['E3P5M0B4']
|
||||
|
||||
|
||||
def test_azure_all_matched_emits_no_assets_file():
|
||||
produced = AzureDeviceGroupSource.transform(
|
||||
AZURE_CSV,
|
||||
options={'hostname_map': {'f13hsgh4': '13NSGH4', 'e3p5m0b4': 'E9Q'}})
|
||||
# Every device matched, so there is nothing to create and no assets file.
|
||||
assert 'assets' not in produced
|
||||
computers = _rows(produced['computers'])
|
||||
assert {c['assetid'] for c in computers} == {'13NSGH4', 'E9Q'}
|
||||
|
||||
|
||||
def test_azure_skips_non_device_rows_and_dedupes():
|
||||
text = (
|
||||
'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n'
|
||||
'F13HSGH4,device,,,1,aaaa,,False,False,False\n'
|
||||
'F13HSGH4,device,,,1,aaaa,,False,False,False\n'
|
||||
'Some User,user,u@x,,2,,u@x,True,False,False\n'
|
||||
)
|
||||
computers = _rows(AzureDeviceGroupSource.transform(text)['computers'])
|
||||
assert [c['hostname'] for c in computers] == ['F13HSGH4']
|
||||
|
||||
|
||||
def test_azure_rejects_wrong_header():
|
||||
with pytest.raises(ImportSourceError):
|
||||
AzureDeviceGroupSource.transform('a,b,c\n1,2,3\n')
|
||||
|
||||
|
||||
THOR_CSV = (
|
||||
'"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n'
|
||||
'"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
|
||||
)
|
||||
|
||||
|
||||
def test_thor_matches_signature():
|
||||
header = ['Host Name', 'BIOS Manf.', 'OS Version', 'Model']
|
||||
assert ThorSource.matches(header)
|
||||
assert detect_source(header) is ThorSource
|
||||
|
||||
|
||||
def test_thor_transform_builds_lookup_and_subtype_files():
|
||||
produced = ThorSource.transform(THOR_CSV)
|
||||
assert 'vendors' in produced
|
||||
assert 'models' in produced
|
||||
assert 'operatingsystems' in produced
|
||||
assert 'assets' in produced
|
||||
assert 'computers' in produced
|
||||
|
||||
assert _rows(produced['vendors'])[0]['vendor'] == 'Dell Inc.'
|
||||
assert _rows(produced['models'])[0]['modelnumber'] == 'Dell Pro Tower Plus QBT1250'
|
||||
assert _rows(produced['operatingsystems'])[0]['osname'] == '10.0.26100 (24H2)'
|
||||
|
||||
computer = _rows(produced['computers'])[0]
|
||||
assert computer['hostname'] == 'F67HSGH4'
|
||||
assert computer['vendorid'] == 'Dell Inc.'
|
||||
assert computer['modelnumberid'] == 'Dell Pro Tower Plus QBT1250'
|
||||
assert computer['osid'] == '10.0.26100 (24H2)'
|
||||
assert computer['loggedinuser'] == 'ShopFloor'
|
||||
assert computer['lastreporteddate'] == '2026-01-02 03:04:05'
|
||||
assert computer['lastboottime'] == '2026-01-01 06:00:00'
|
||||
|
||||
asset = _rows(produced['assets'])[0]
|
||||
assert asset['locationid'] == 'S13'
|
||||
assert asset['assettypeid'] == 'computer'
|
||||
assert asset['statusid'] == 'In Use'
|
||||
|
||||
|
||||
THOR_TWO_CSV = (
|
||||
'"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n'
|
||||
'"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
|
||||
'"FDKKSGH4","10.0.0.6","FDKKSGH4\\ShopFloor","10.0.26100","24H2","2026-01-03 03:04:05","2026-01-02 06:00:00","G8","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
|
||||
)
|
||||
|
||||
|
||||
def test_thor_stamps_computertype():
|
||||
computer = _rows(
|
||||
ThorSource.transform(THOR_CSV, options={'computertype': 'Shop Floor PC'})['computers'])[0]
|
||||
assert computer['computertypeid'] == 'Shop Floor PC'
|
||||
|
||||
|
||||
def test_thor_aligns_matched_hostname_to_existing_asset():
|
||||
# A case-insensitive hostname match routes the host to its real asset and
|
||||
# emits NO new asset row, so the import updates in place instead of creating
|
||||
# a duplicate. Unmatched hosts still create a new asset keyed on hostname.
|
||||
produced = ThorSource.transform(
|
||||
THOR_TWO_CSV, options={'hostname_map': {'f67hsgh4': 'PC-1001'}})
|
||||
|
||||
computers = _rows(produced['computers'])
|
||||
by_host = {c['hostname']: c for c in computers}
|
||||
assert by_host['F67HSGH4']['assetid'] == 'PC-1001'
|
||||
assert by_host['FDKKSGH4']['assetid'] == 'FDKKSGH4'
|
||||
|
||||
assets = _rows(produced['assets'])
|
||||
assert [a['assetnumber'] for a in assets] == ['FDKKSGH4']
|
||||
|
||||
|
||||
def test_thor_all_matched_emits_no_assets_file():
|
||||
produced = ThorSource.transform(
|
||||
THOR_TWO_CSV,
|
||||
options={'hostname_map': {'f67hsgh4': 'PC-1001', 'fdkksgh4': 'PC-1002'}})
|
||||
assert 'assets' not in produced
|
||||
computers = _rows(produced['computers'])
|
||||
assert {c['assetid'] for c in computers} == {'PC-1001', 'PC-1002'}
|
||||
|
||||
|
||||
def test_get_source_unknown_raises():
|
||||
with pytest.raises(ImportSourceError):
|
||||
get_source('nope')
|
||||
|
||||
|
||||
def test_detect_source_none_for_unknown_header():
|
||||
assert detect_source(['foo', 'bar']) is None
|
||||
@@ -46,6 +46,10 @@ CUTOVER_PLUGINS = (
|
||||
# stamp '<plugin>0001anchor'; measuringtools stamps its real baseline id.
|
||||
EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS}
|
||||
EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline'
|
||||
# computers adds the Entra/Azure device identity columns (azuredeviceid, then
|
||||
# azureobjectid + azuredevicecategory), then the Configuration/Business policy
|
||||
# tables, on top of its anchor, so its head is not the f-string default.
|
||||
EXPECTED_HEAD_REVISION['computers'] = 'computers0004policies'
|
||||
# backups is also post-cutover: its 0001 really creates backuprevisions, and
|
||||
# 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup
|
||||
# otherwise throws away.
|
||||
@@ -83,6 +87,11 @@ EXPECTED_HEAD_REVISION['tools'] = 'tools0001securesend'
|
||||
EXPECTED_HEAD_REVISION['notifications'] = 'notifications0005boardorder'
|
||||
# warranty adds the proof-of-cover document columns on top of its anchor.
|
||||
EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof'
|
||||
# endpoints is a NEW post-cutover plugin: its 0001 genuinely creates the
|
||||
# endpointstates + endpointevents tables the core chain never owned, 0002
|
||||
# adds the endpointcommands queue plus the MTConnect state columns, and 0003
|
||||
# adds the generic command target/targetkind columns.
|
||||
EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0003cmdtarget'
|
||||
|
||||
# Plugins built after the cutover: their 0001 baseline really creates tables the
|
||||
# core chain never owned.
|
||||
|
||||
187
tests/test_plugins/test_computers_import_machine_link.py
Normal file
187
tests/test_plugins/test_computers_import_machine_link.py
Normal file
@@ -0,0 +1,187 @@
|
||||
"""computers.csv can carry the machine it drives, and the link is built on import.
|
||||
|
||||
A site with a spreadsheet knows "this PC drives machine CNC-014" and wants to
|
||||
write that in the PC row, not hand-build an assetrelationships sheet keyed on
|
||||
ids it never sees. The importer accepts a virtual `machineassetnumber` column,
|
||||
keeps it out of the computers write, and hands it to the computers plugin after
|
||||
each PC row is written. The rules that matter for a cold user driving this off a
|
||||
spreadsheet:
|
||||
|
||||
- a good number builds the PC-controls-machine link;
|
||||
- a number that has no machine, names this PC itself, names a location-only
|
||||
marker, or is claimed twice in one file, is an AMBER warning and is skipped -
|
||||
the PC still imports, the file is still ok;
|
||||
- a dry run builds nothing and, critically, sends no alert;
|
||||
- re-running the same file does not duplicate the link.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from shopdb.core.services.csvimport import (
|
||||
Resolver, import_csv, virtual_columns_for,
|
||||
)
|
||||
from shopdb.extensions import db as _db
|
||||
|
||||
|
||||
REFERENCE = [
|
||||
('assettypes', 'assettype\nComputer\nMachine\n'),
|
||||
('assetstatuses', 'status\nIn Use\n'),
|
||||
('locationtypes', 'locationtype\nBay\n'),
|
||||
]
|
||||
|
||||
|
||||
def _seed(resolver):
|
||||
"""Reference rows, a location, and the 'controls' relationship type."""
|
||||
from shopdb.core.models.relationship import RelationshipType
|
||||
|
||||
for table, text in REFERENCE:
|
||||
result = import_csv(table, text, resolver=resolver, commit=True)
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
result = import_csv(
|
||||
'locations', 'locationname,locationtypeid\nBay 3,Bay\n',
|
||||
resolver=resolver, commit=True)
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
if not RelationshipType.query.filter_by(relationshiptype='controls').first():
|
||||
_db.session.add(RelationshipType(relationshiptype='controls'))
|
||||
# import_csv writes into the session but never commits; commit the setup so
|
||||
# a dry-run import's rollback only unwinds the dry run, not the fixtures.
|
||||
_db.session.commit()
|
||||
|
||||
|
||||
def _assets(resolver, *rows):
|
||||
"""Create the given assets. Each row is (assetnumber, assettype)."""
|
||||
body = 'assetnumber,assettypeid,statusid,locationid\n'
|
||||
body += ''.join('%s,%s,In Use,Bay 3\n' % r for r in rows)
|
||||
result = import_csv('assets', body, resolver=resolver, commit=True)
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
_db.session.commit()
|
||||
|
||||
|
||||
def _import_computers(resolver, body, commit=True):
|
||||
return import_csv('computers', body, resolver=resolver, commit=commit)
|
||||
|
||||
|
||||
def _controls_target(pc_assetnumber):
|
||||
"""Asset number of the machine this PC actively controls, or None."""
|
||||
from shopdb.core.models import Asset
|
||||
from shopdb.core.models.relationship import RelationshipType, AssetRelationship
|
||||
|
||||
pc = Asset.query.filter_by(assetnumber=pc_assetnumber).first()
|
||||
controls = RelationshipType.query.filter_by(relationshiptype='controls').first()
|
||||
rel = AssetRelationship.query.filter(
|
||||
AssetRelationship.sourceassetid == pc.assetid,
|
||||
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
|
||||
AssetRelationship.isactive.is_(True)).first()
|
||||
if rel is None:
|
||||
return None
|
||||
return _db.session.get(Asset, rel.targetassetid).assetnumber
|
||||
|
||||
|
||||
def test_the_column_is_virtual_and_owned_by_the_computers_plugin(db):
|
||||
"""The plugin registers machineassetnumber; core stays ignorant of it."""
|
||||
assert 'machineassetnumber' in virtual_columns_for('computers')
|
||||
|
||||
|
||||
def test_a_good_machine_number_builds_the_link(db):
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
|
||||
result = _import_computers(
|
||||
resolver,
|
||||
'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n')
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
assert result.warnings == []
|
||||
assert _controls_target('PC-001') == 'CNC-014'
|
||||
|
||||
|
||||
def test_an_empty_machine_number_links_nothing_and_does_not_warn(db):
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver, ('PC-001', 'Computer'))
|
||||
result = _import_computers(
|
||||
resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,\n')
|
||||
assert result.ok
|
||||
assert result.warnings == []
|
||||
assert _controls_target('PC-001') is None
|
||||
|
||||
|
||||
def test_a_machine_number_with_no_machine_warns_and_skips(db):
|
||||
"""The PC still imports; the unresolved number is amber, not a failed file."""
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver, ('PC-001', 'Computer'))
|
||||
result = _import_computers(
|
||||
resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,NOPE-99\n')
|
||||
assert result.ok, 'an unresolved machine number must not fail the file'
|
||||
assert len(result.warnings) == 1
|
||||
warning = result.warnings[0]
|
||||
assert warning.column == 'machineassetnumber'
|
||||
assert warning.target == 'NOPE-99'
|
||||
assert warning.action, 'a warning must carry the one corrective step'
|
||||
assert _controls_target('PC-001') is None
|
||||
|
||||
|
||||
def test_a_pc_naming_its_own_asset_number_is_not_self_linked(db):
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver, ('PC-001', 'Computer'))
|
||||
result = _import_computers(
|
||||
resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,PC-001\n')
|
||||
assert result.ok
|
||||
assert len(result.warnings) == 1
|
||||
assert 'own asset number' in result.warnings[0].reason
|
||||
assert _controls_target('PC-001') is None
|
||||
|
||||
|
||||
def test_two_rows_claiming_one_machine_link_only_the_first(db):
|
||||
"""The in-file fan-out guard: a copied cell must not steal the bay."""
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver,
|
||||
('PC-001', 'Computer'), ('PC-002', 'Computer'), ('CNC-014', 'Machine'))
|
||||
result = _import_computers(
|
||||
resolver,
|
||||
'assetid,hostname,machineassetnumber\n'
|
||||
'PC-001,WS-PC-001,CNC-014\n'
|
||||
'PC-002,WS-PC-002,CNC-014\n')
|
||||
assert result.ok
|
||||
assert len(result.warnings) == 1
|
||||
assert 'already claimed' in result.warnings[0].reason
|
||||
assert _controls_target('PC-001') == 'CNC-014'
|
||||
assert _controls_target('PC-002') is None
|
||||
|
||||
|
||||
def test_a_dry_run_builds_no_link_and_leaves_no_trace(db):
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
|
||||
result = _import_computers(
|
||||
resolver,
|
||||
'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n',
|
||||
commit=False)
|
||||
assert result.ok
|
||||
assert result.created == 1
|
||||
_db.session.rollback()
|
||||
assert _controls_target('PC-001') is None
|
||||
|
||||
|
||||
def test_reimporting_the_same_link_does_not_duplicate(db):
|
||||
resolver = Resolver()
|
||||
_seed(resolver)
|
||||
_assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
|
||||
body = 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n'
|
||||
first = _import_computers(resolver, body)
|
||||
assert first.ok and first.warnings == []
|
||||
|
||||
second = _import_computers(resolver, body)
|
||||
assert second.ok and second.warnings == []
|
||||
assert _controls_target('PC-001') == 'CNC-014'
|
||||
|
||||
from shopdb.core.models import Asset
|
||||
from shopdb.core.models.relationship import RelationshipType, AssetRelationship
|
||||
pc = Asset.query.filter_by(assetnumber='PC-001').first()
|
||||
controls = RelationshipType.query.filter_by(relationshiptype='controls').first()
|
||||
active = AssetRelationship.query.filter(
|
||||
AssetRelationship.sourceassetid == pc.assetid,
|
||||
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
|
||||
AssetRelationship.isactive.is_(True)).count()
|
||||
assert active == 1, 'a re-import must not stack a second active link'
|
||||
Reference in New Issue
Block a user