902339baf9
The endpoint agent already reports an installed-software list (SBOM), but ingress dropped it: the collector schema had no entry for it and there was nowhere to store it, so operators could not see what is installed on a PC. Persist it as a per-asset snapshot in a new endpointapplications table, replaced on every report (current inventory is what operators ask for, not install history). Add applications to the collector schema and KNOWN_PAYLOAD_KEYS so ingress stops discarding it, a read route for the raw collection, and a collapsed-by-default Installed software panel to surface it. appid is nullable groundwork for a future Managed Applications link; no matcher is wired yet.
268 lines
12 KiB
Python
268 lines
12 KiB
Python
"""Shared Alembic env.py logic for bundled plugins.
|
|
|
|
Every bundled plugin that owns tables (computers, employees, knowledgebase,
|
|
machines, network, notifications, printers, slides, usb, warranty) has a
|
|
`migrations/env.py` that does the minimum:
|
|
|
|
import os
|
|
os.environ['PLUGIN_NAME'] = 'computers'
|
|
from shopdb.plugins.alembic_template import run_migrations
|
|
run_migrations()
|
|
|
|
This module wires the plugin's models into a MetaData object filtered to
|
|
only the tables that belong to that plugin, then runs Alembic in either
|
|
offline or online mode against the Flask app's configured engine.
|
|
|
|
Ownership cutover (see ADR-008): the core Alembic chain created every table
|
|
that exists through its head (`7d16_directoryemployees`), including the plugin
|
|
tables. Each plugin's `0001` migration is therefore a stamp-only no-op that
|
|
just records the anchor revision in `alembic_version_<plugin>`. NEW plugin
|
|
schema changes land as `plugins/<name>/migrations/000N` from here on, never in
|
|
the core chain.
|
|
|
|
Plugin tables must be importable via `plugins.<name>.models`. Plugins
|
|
register their `__tablename__` set in PLUGIN_TABLE_OWNERS below so the
|
|
filter is explicit (avoids depending on import-side-effect global state).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import logging
|
|
import os
|
|
from typing import Iterable
|
|
|
|
from alembic import context
|
|
from sqlalchemy import MetaData, pool
|
|
|
|
# Registers the compiler hook that forces utf8mb4 + DYNAMIC on MySQL. Imported
|
|
# for the side effect. It used to live inline in migrations/env.py, so it applied
|
|
# to the CORE chain only: a plugin's baseline tables were created at the server's
|
|
# default charset while core's were utf8mb4, on the same database. On a server
|
|
# defaulting to latin1 that difference is invisible until a join between the two
|
|
# stops using an index, or a character comes back mangled.
|
|
import shopdb.utils.mysql_charset # noqa: F401,E402
|
|
|
|
logger = logging.getLogger('alembic.env.plugin')
|
|
|
|
# Explicit table-ownership map. Adding tables to a plugin requires updating
|
|
# this dict so the per-plugin migration knows which tables to include.
|
|
PLUGIN_TABLE_OWNERS: dict[str, Iterable[str]] = {
|
|
'backups': ('backuprevisions',),
|
|
'computers': ('computertypes', 'computers', 'computerinstalledapps',
|
|
'accessprotocols', 'computeraccess',
|
|
'configurationpolicies', 'configurationpolicyactions',
|
|
'businesspolicies', 'businesspolicyitems'),
|
|
'employees': ('directoryemployees',),
|
|
'endpoints': ('endpointstates', 'endpointevents', 'endpointcommands', 'endpointapplications'),
|
|
'geenforce': ('manifestscopes', 'manifestentries', 'manifestentrypctypes',
|
|
'manifestentryhostnames', 'manifestentrymachinenumbers',
|
|
'manifestinusechecks', 'manifestinusecheckprocesses',
|
|
'manifestpublishedversions', 'manifestpayloads',
|
|
'manifestblobs',
|
|
'manifestenforcementreports', 'manifestenforcementresults',
|
|
'pctypealiases'),
|
|
'guides': ('guides', 'guiderevisions', 'guideimages'),
|
|
'knowledgebase': ('knowledgebase',),
|
|
'machines': ('machinetypes', 'machines'),
|
|
'measuringtools': ('measuringtooltypes', 'measuringtools'),
|
|
'network': ('networkdevicetypes', 'networkdevices', 'vlans', 'subnets'),
|
|
'notifications': ('notificationtypes', 'notifications'),
|
|
'printedparts': ('printeditems', 'printeditemtransactions',
|
|
'printeditemfiles'),
|
|
'printers': ('printertypes', 'printers', 'modelsupplies', 'printerdrivers',
|
|
'printersupplyalerts', 'printerobservedqueues'),
|
|
'slides': ('tvslides',),
|
|
# Tech Tools was schemaless until secure send: every other tool in it
|
|
# runs in the browser and owns nothing.
|
|
'tools': ('securesendsecrets',),
|
|
'usb': ('usbdevicetypes', 'usbdevices', 'usbcheckouts'),
|
|
'warranty': ('warranties', 'warrantyassets'),
|
|
}
|
|
|
|
|
|
def _get_plugin_metadata(plugin_name: str) -> MetaData:
|
|
"""Import the plugin's models and return a MetaData containing only its
|
|
declared tables (filtered via PLUGIN_TABLE_OWNERS)."""
|
|
owned = set(PLUGIN_TABLE_OWNERS.get(plugin_name, ()))
|
|
if not owned:
|
|
raise RuntimeError(
|
|
f"PLUGIN_TABLE_OWNERS has no entry for plugin '{plugin_name}'. "
|
|
f"Update shopdb/plugins/alembic_template.py."
|
|
)
|
|
|
|
# Importing models attaches them to the global db.metadata.
|
|
importlib.import_module(f'plugins.{plugin_name}.models')
|
|
from shopdb.extensions import db
|
|
full = db.metadata
|
|
|
|
plugin_md = MetaData()
|
|
for table in list(full.tables.values()):
|
|
if table.name in owned:
|
|
table.to_metadata(plugin_md)
|
|
return plugin_md
|
|
|
|
|
|
def create_plugin_tables(plugin_name: str):
|
|
"""Create every table this plugin owns, sourced from the SQLAlchemy models
|
|
(not duplicated DDL). IDEMPOTENT: a table that already exists is skipped, so
|
|
this is safe on an existing database that has the table from the pre-cutover
|
|
core baseline as well as on a fresh install (ADR-014 Phase 2).
|
|
|
|
NOT actually called from anywhere: every post-cutover baseline emits
|
|
explicit Alembic ops instead, because this helper's per-plugin MetaData
|
|
cannot resolve a foreign key to a CORE table at CreateTable-compile time.
|
|
Kept because it is still the reference for what idempotent means here, and
|
|
because create_table_if_missing below is the same rule in a form the
|
|
baselines can use. See that function for what went wrong without it.
|
|
"""
|
|
from alembic import op
|
|
from sqlalchemy import inspect
|
|
from sqlalchemy.schema import CreateTable
|
|
|
|
md = _get_plugin_metadata(plugin_name)
|
|
bind = op.get_bind()
|
|
existing = set(inspect(bind).get_table_names())
|
|
# Sort by FK dependency so parent tables are created first.
|
|
for table in md.sorted_tables:
|
|
if table.name in existing:
|
|
continue
|
|
op.execute(str(CreateTable(table).compile(dialect=bind.dialect)))
|
|
|
|
|
|
def create_table_if_missing(table_name: str, *columns, **kwargs) -> bool:
|
|
"""op.create_table, skipped when the table is already there.
|
|
|
|
WHY EVERY POST-CUTOVER BASELINE NEEDS THIS. A baseline is the sole creator
|
|
of its plugin's tables, so it was written as a plain op.create_table - which
|
|
is correct only if the table's existence and the plugin's Alembic stamp can
|
|
never disagree. They can, and routinely do:
|
|
|
|
* `flask db-utils create-all` runs db.create_all(), which builds EVERY
|
|
table registered on the metadata - plugin tables included - and stamps
|
|
NOTHING. `flask plugin upgrade-all` then runs the baseline against
|
|
tables that already exist.
|
|
* A run that creates the table and then fails before its stamp commits
|
|
leaves the same split, and re-running is the obvious thing to try.
|
|
|
|
Reported as `flask plugin upgrade-all` failing the tools plugin with
|
|
"table securesendsecrets already exists". Nothing caught it because the
|
|
migration test drops the post-cutover tables db.create_all() made before
|
|
running upgrade-all, which arranges the one state where an unguarded create
|
|
works and never the state a real database is in.
|
|
|
|
Returns True when it created the table, so a caller can skip the indexes
|
|
that belong to it.
|
|
"""
|
|
from alembic import op
|
|
from sqlalchemy import inspect
|
|
|
|
if inspect(op.get_bind()).has_table(table_name):
|
|
logger.info('%s already exists, skipping create', table_name)
|
|
return False
|
|
op.create_table(table_name, *columns, **kwargs)
|
|
return True
|
|
|
|
|
|
def create_index_if_missing(index_name: str, table_name: str, columns, **kwargs):
|
|
"""op.create_index, skipped when an equivalent index already exists.
|
|
|
|
Matched on the COLUMN LIST rather than the name, because the two creators
|
|
do not agree on names. db.create_all() takes them from SQLAlchemy's default
|
|
convention (`ix_securesendsecrets_token`) while a baseline spells them the
|
|
way this repo's naming rules require (`ixsecuresendsecretstoken`). Matching
|
|
on the name would call the index missing and add a second one over the same
|
|
column - MySQL permits that, with only a warning, so the duplicate would
|
|
accumulate silently on every affected site.
|
|
"""
|
|
from alembic import op
|
|
from sqlalchemy import inspect
|
|
|
|
wanted = [str(column) for column in columns]
|
|
inspector = inspect(op.get_bind())
|
|
if not inspector.has_table(table_name):
|
|
op.create_index(index_name, table_name, columns, **kwargs)
|
|
return
|
|
for existing in inspector.get_indexes(table_name):
|
|
if list(existing.get('column_names') or []) == wanted:
|
|
logger.info('index over %s(%s) already exists as %s, skipping',
|
|
table_name, ', '.join(wanted), existing.get('name'))
|
|
return
|
|
op.create_index(index_name, table_name, columns, **kwargs)
|
|
|
|
|
|
def drop_plugin_tables(plugin_name: str):
|
|
"""Mirror of create_plugin_tables for downgrade(). Drops in reverse FK
|
|
order."""
|
|
from alembic import op
|
|
|
|
md = _get_plugin_metadata(plugin_name)
|
|
for table in reversed(md.sorted_tables):
|
|
op.execute(f'DROP TABLE IF EXISTS "{table.name}"')
|
|
|
|
|
|
def run_migrations():
|
|
"""Entry point called by each plugin's migrations/env.py."""
|
|
plugin_name = os.environ.get('PLUGIN_NAME')
|
|
if not plugin_name:
|
|
raise RuntimeError("PLUGIN_NAME env var must be set before run_migrations()")
|
|
|
|
config = context.config
|
|
target_metadata = _get_plugin_metadata(plugin_name)
|
|
|
|
# Per-plugin version table so each plugin's chain is independent of core
|
|
# Alembic's alembic_version table.
|
|
version_table = f'alembic_version_{plugin_name}'
|
|
|
|
db_url = config.get_main_option('sqlalchemy.url')
|
|
if not db_url:
|
|
# Pull from the Flask app config if running inside an app context
|
|
# (e.g. via flask plugin migrate <name>).
|
|
try:
|
|
from flask import current_app
|
|
db_url = current_app.config['SQLALCHEMY_DATABASE_URI']
|
|
config.set_main_option('sqlalchemy.url', db_url.replace('%', '%%'))
|
|
except Exception as ex:
|
|
raise RuntimeError(
|
|
"sqlalchemy.url not set and no Flask app context available. "
|
|
f"Original error: {ex}"
|
|
)
|
|
|
|
if context.is_offline_mode():
|
|
context.configure(
|
|
url=db_url,
|
|
target_metadata=target_metadata,
|
|
literal_binds=True,
|
|
version_table=version_table,
|
|
include_schemas=False,
|
|
)
|
|
with context.begin_transaction():
|
|
context.run_migrations()
|
|
else:
|
|
# Build the engine straight from the resolved URL. The plugin manager
|
|
# drives this via a programmatic alembic Config (no ini file), so
|
|
# config.get_section returns an empty dict and engine_from_config would
|
|
# find no sqlalchemy.url. db_url is already resolved above.
|
|
from sqlalchemy import create_engine
|
|
connectable = create_engine(db_url, poolclass=pool.NullPool)
|
|
with connectable.connect() as connection:
|
|
context.configure(
|
|
connection=connection,
|
|
target_metadata=target_metadata,
|
|
version_table=version_table,
|
|
include_schemas=False,
|
|
)
|
|
with context.begin_transaction():
|
|
context.run_migrations()
|
|
# SAME FIX AS migrations/env.py. MySQL DDL commits implicitly, which
|
|
# flushes everything queued before it - including the PREVIOUS
|
|
# migration's version stamp. The LAST migration of a run has no DDL
|
|
# after it, so its stamp is rolled back when the connection closes:
|
|
# the schema change survives, alembic_version stays one revision
|
|
# behind, and `flask plugin upgrade-all` exits 0 having silently
|
|
# re-run that migration - again on every deploy after.
|
|
#
|
|
# Core got this fix; the per-plugin chains run through THIS file and
|
|
# did not, so every plugin head has been re-running on MySQL. That is
|
|
# only invisible while every head happens to be idempotent.
|
|
connection.commit()
|