Bring the wilm branch into main: endpoints plugin, agent, policies, CSV import
Ports the corporate-GitHub wilm branch onto main, keeping the two branches
separate. Thirteen upstream commits arrive with their authorship intact; the
review fixes and the portability work below are on top.
What arrives:
endpoints plugin - a ShopDB agent reporting endpoint state, telemetry,
services and interfaces, with a command channel the agent
long-polls. Ingest stays on the generic collector route
(ADR-006); the agent key is a UI-managed single-scope
token whose revocation is an instant kill switch.
computers policies - Configuration and Business Policies (RMM-style), scope
trees validated at write time, actions checked against
the endpoints command allowlist. Feature-gated: every
route 404s while the setting is off, so enabling it can
never half-expose the surface.
asset CSV import - templates, a collector script carrying the site path, and
a validate-then-commit import. A plugin can own a virtual
column plus a post-write row hook, which is how a PC's
machine number becomes a controls link without a
hand-built relationships sheet.
agent/ - the Windows agent and its installer.
Contract 0.21.0 -> 0.24.0, additive throughout: register_import_hook (0.22),
mint/list/revoke_service_token (0.23), collector_authorized (0.24). The stability
table had no rows for any of them and now does.
MySQL 5.6 portability. The branch stored three policy columns as sa.JSON, the
only JSON columns in the product, and 5.6 has no JSON type: the migration failed
with a bare syntax error near the word JSON. They are TEXT now, through a
JSONText type decorator, so all seventeen call sites still read and write dicts.
The convention elsewhere is already TEXT (guides.blocksjson,
geenforce.manifestjson).
Review fixes carried on top:
- six policy GETs were jwt_required(optional=True), which authenticates a
caller without checking anything; they require computers.view now, and six
redundant jwt_required decorators are gone
- the agent installer took a default BaseUrl and CollectorKey, so a mistyped
run silently pointed an endpoint at the wrong instance; both are mandatory
and a non-https BaseUrl throws unless -AllowInsecureHttp is passed
- a committing CSV import wrote no audit row
Documentation: the 34 new routes are described in docs/api-inventory.json, which
is hand-maintained on purpose, and the generated spec now covers 331 paths and
477 operations across 22 surfaces. DEVELOPMENT-SETUP's 1071 troubleshooting row
said "use MySQL 8 for dev" and now gives the actual fix, including the trap that
5.6 downgrades ROW_FORMAT=DYNAMIC to Compact with a warning instead of an error.
Full suite 2123 passed, vitest 260 passed, all seventeen plugin chains migrate
clean on MySQL 5.6 and the naming, docs, parity and read-surface gates are green.
This commit is contained in:
@@ -12,6 +12,7 @@ from shopdb.core.services.csvimport import (
|
||||
IMPORTABLE, ImportError_, Resolver, generate_template, import_csv,
|
||||
importable_columns, required_columns,
|
||||
)
|
||||
from shopdb.core.services.csvimport import digit_runs, normalize_label
|
||||
from shopdb.extensions import db as _db
|
||||
|
||||
|
||||
@@ -173,14 +174,17 @@ def test_generated_templates_only_contain_real_columns(db):
|
||||
|
||||
A hand-written template set had invented columns on 7 of 11 tables and named
|
||||
a table that does not exist. Generating from the schema is what makes that
|
||||
impossible - so assert the generator actually does it.
|
||||
impossible - so assert the generator actually does it. Plugin-registered
|
||||
virtual columns (machineassetnumber) are legitimate, not invented, so they
|
||||
are allowed alongside the table's real columns.
|
||||
"""
|
||||
from shopdb.core.services.csvimport import virtual_columns_for
|
||||
for table in IMPORTABLE:
|
||||
text = generate_template(table)
|
||||
header = [line for line in text.splitlines() if line and not line.startswith('#')][0]
|
||||
real = {c.name for c in importable_columns(table)}
|
||||
allowed = {c.name for c in importable_columns(table)} | virtual_columns_for(table)
|
||||
for column in header.split(','):
|
||||
assert column in real, '%s.csv offers a column that does not exist: %s' % (table, column)
|
||||
assert column in allowed, '%s.csv offers a column that does not exist: %s' % (table, column)
|
||||
|
||||
|
||||
def test_every_template_includes_its_required_columns(db):
|
||||
@@ -234,3 +238,100 @@ def test_a_dry_run_over_several_files_resolves_across_them(db):
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
_db.session.rollback()
|
||||
assert count('assets') == 0
|
||||
|
||||
|
||||
def _example_files():
|
||||
"""Reconstruct each template's example rows as an importable CSV: the header
|
||||
plus the live (uncommented) example data rows that follow it."""
|
||||
files = {}
|
||||
for table in IMPORTABLE:
|
||||
header = None
|
||||
rows = []
|
||||
for line in generate_template(table).splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
if line.startswith('#'):
|
||||
continue
|
||||
if header is None:
|
||||
header = line
|
||||
else:
|
||||
rows.append(line)
|
||||
files[table] = '\n'.join([header] + rows) + '\n'
|
||||
return files
|
||||
|
||||
|
||||
def test_template_examples_are_present_for_every_table(db):
|
||||
"""Each template ships at least one worked example row, so a downloaded
|
||||
set is a usable starter workbook rather than a bare header."""
|
||||
files = _example_files()
|
||||
for table in IMPORTABLE:
|
||||
rows = files[table].splitlines()[1:]
|
||||
assert rows, '%s.csv has no example row' % table
|
||||
|
||||
|
||||
def test_template_example_set_imports_cleanly(db):
|
||||
"""The whole curated example set must import as-is: every name a later file
|
||||
references is defined by an earlier one, with no self-references that the
|
||||
validation pass cannot resolve. This is the promise the workbook makes."""
|
||||
resolver = Resolver()
|
||||
files = _example_files()
|
||||
for table in IMPORTABLE:
|
||||
result = import_csv(table, files[table], resolver=resolver)
|
||||
assert result.ok, '%s.csv example failed: %s' % (
|
||||
table, [str(p) for p in result.problems])
|
||||
_db.session.rollback()
|
||||
|
||||
|
||||
# -- fuzzy reconciliation safety (review item 1.1) -----------------------------
|
||||
#
|
||||
# A high similarity score is the DANGEROUS case: 'Bay 310' and 'Bay 311' share
|
||||
# every word and score ~0.857, so a naive threshold silently merges two distinct
|
||||
# locations. Reconciliation now ships OFF (threshold 0) and, even when a site
|
||||
# turns it on, refuses any pair whose digit runs differ.
|
||||
|
||||
def test_digit_runs_extracts_number_groups():
|
||||
assert digit_runs(normalize_label('Bay 310')) == ['310']
|
||||
assert digit_runs(normalize_label('Precision 3660')) == ['3660']
|
||||
assert digit_runs(normalize_label('Dell Inc.')) == []
|
||||
|
||||
|
||||
def test_reconciliation_is_off_by_default(db):
|
||||
"""A fresh Resolver does not fuzzy-match: an unknown numbered bay is a row
|
||||
error, not a silent bind onto its neighbour."""
|
||||
resolver = Resolver()
|
||||
seed_reference(resolver)
|
||||
# 'Bay 3' exists from the reference seed; 'Bay 8' does not.
|
||||
result = import_csv(
|
||||
'assets',
|
||||
'assetnumber,assettypeid,locationid\nA-1,Machine,Bay 8\n',
|
||||
resolver=resolver)
|
||||
assert not result.ok
|
||||
assert not resolver.events
|
||||
|
||||
|
||||
def test_reconcile_refuses_when_digit_runs_differ(db):
|
||||
"""Even with fuzzy matching turned on, 'Bay 311' must not bind to the
|
||||
existing 'Bay 3': the digit runs differ, so it is a distinct location."""
|
||||
resolver = Resolver(reconcile_threshold=0.5)
|
||||
seed_reference(resolver)
|
||||
result = import_csv(
|
||||
'assets',
|
||||
'assetnumber,assettypeid,locationid\nB-1,Machine,Bay 311\n',
|
||||
resolver=resolver)
|
||||
assert not result.ok
|
||||
assert not resolver.events
|
||||
|
||||
|
||||
def test_reconcile_still_binds_normalized_exact(db):
|
||||
"""The normalized-exact tier is untouched: 'bay 3' folds to the existing
|
||||
'Bay 3' regardless of threshold, and is reported as an event."""
|
||||
resolver = Resolver(reconcile_threshold=0.9)
|
||||
seed_reference(resolver)
|
||||
result = import_csv(
|
||||
'assets',
|
||||
'assetnumber,assettypeid,locationid\nC-1,Machine,BAY 3\n',
|
||||
resolver=resolver)
|
||||
assert result.ok, [str(p) for p in result.problems]
|
||||
_db.session.rollback()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user