"""Shared Alembic env.py logic for bundled plugins. Every bundled plugin that owns tables (computers, employees, knowledgebase, machines, network, notifications, printers, slides, usb, warranty) has a `migrations/env.py` that does the minimum: import os os.environ['PLUGIN_NAME'] = 'computers' from shopdb.plugins.alembic_template import run_migrations run_migrations() This module wires the plugin's models into a MetaData object filtered to only the tables that belong to that plugin, then runs Alembic in either offline or online mode against the Flask app's configured engine. Ownership cutover (see ADR-008): the core Alembic chain created every table that exists through its head (`7d16_directoryemployees`), including the plugin tables. Each plugin's `0001` migration is therefore a stamp-only no-op that just records the anchor revision in `alembic_version_`. NEW plugin schema changes land as `plugins//migrations/000N` from here on, never in the core chain. Plugin tables must be importable via `plugins..models`. Plugins register their `__tablename__` set in PLUGIN_TABLE_OWNERS below so the filter is explicit (avoids depending on import-side-effect global state). """ from __future__ import annotations import importlib import logging import os from typing import Iterable from alembic import context from sqlalchemy import MetaData, pool # Registers the compiler hook that forces utf8mb4 + DYNAMIC on MySQL. Imported # for the side effect. It used to live inline in migrations/env.py, so it applied # to the CORE chain only: a plugin's baseline tables were created at the server's # default charset while core's were utf8mb4, on the same database. On a server # defaulting to latin1 that difference is invisible until a join between the two # stops using an index, or a character comes back mangled. import shopdb.utils.mysql_charset # noqa: F401,E402 logger = logging.getLogger('alembic.env.plugin') # Explicit table-ownership map. Adding tables to a plugin requires updating # this dict so the per-plugin migration knows which tables to include. PLUGIN_TABLE_OWNERS: dict[str, Iterable[str]] = { 'backups': ('backuprevisions',), 'computers': ('computertypes', 'computers', 'computerinstalledapps', 'accessprotocols', 'computeraccess', 'configurationpolicies', 'configurationpolicyactions', 'businesspolicies', 'businesspolicyitems'), 'employees': ('directoryemployees',), 'endpoints': ('endpointstates', 'endpointevents', 'endpointcommands', 'endpointapplications', 'endpointmonitors', 'operatorreports'), 'geenforce': ('manifestscopes', 'manifestentries', 'manifestentrypctypes', 'manifestentryhostnames', 'manifestentrymachinenumbers', 'manifestinusechecks', 'manifestinusecheckprocesses', 'manifestpublishedversions', 'manifestpayloads', 'manifestblobs', 'manifestenforcementreports', 'manifestenforcementresults', 'pctypealiases'), 'guides': ('guides', 'guiderevisions', 'guideimages'), 'knowledgebase': ('knowledgebase',), 'machines': ('machinetypes', 'machines'), 'measuringtools': ('measuringtooltypes', 'measuringtools'), 'network': ('networkdevicetypes', 'networkdevices', 'vlans', 'subnets'), 'notifications': ('notificationtypes', 'notifications'), 'printedparts': ('printeditems', 'printeditemtransactions', 'printeditemfiles'), 'printers': ('printertypes', 'printers', 'modelsupplies', 'printerdrivers', 'printersupplyalerts', 'printerobservedqueues'), 'slides': ('tvslides',), # Tech Tools was schemaless until secure send: every other tool in it # runs in the browser and owns nothing. 'tools': ('securesendsecrets',), 'usb': ('usbdevicetypes', 'usbdevices', 'usbcheckouts'), 'warranty': ('warranties', 'warrantyassets'), } def _get_plugin_metadata(plugin_name: str) -> MetaData: """Import the plugin's models and return a MetaData containing only its declared tables (filtered via PLUGIN_TABLE_OWNERS).""" owned = set(PLUGIN_TABLE_OWNERS.get(plugin_name, ())) if not owned: raise RuntimeError( f"PLUGIN_TABLE_OWNERS has no entry for plugin '{plugin_name}'. " f"Update shopdb/plugins/alembic_template.py." ) # Importing models attaches them to the global db.metadata. importlib.import_module(f'plugins.{plugin_name}.models') from shopdb.extensions import db full = db.metadata plugin_md = MetaData() for table in list(full.tables.values()): if table.name in owned: table.to_metadata(plugin_md) return plugin_md def create_plugin_tables(plugin_name: str): """Create every table this plugin owns, sourced from the SQLAlchemy models (not duplicated DDL). IDEMPOTENT: a table that already exists is skipped, so this is safe on an existing database that has the table from the pre-cutover core baseline as well as on a fresh install (ADR-014 Phase 2). NOT actually called from anywhere: every post-cutover baseline emits explicit Alembic ops instead, because this helper's per-plugin MetaData cannot resolve a foreign key to a CORE table at CreateTable-compile time. Kept because it is still the reference for what idempotent means here, and because create_table_if_missing below is the same rule in a form the baselines can use. See that function for what went wrong without it. """ from alembic import op from sqlalchemy import inspect from sqlalchemy.schema import CreateTable md = _get_plugin_metadata(plugin_name) bind = op.get_bind() existing = set(inspect(bind).get_table_names()) # Sort by FK dependency so parent tables are created first. for table in md.sorted_tables: if table.name in existing: continue op.execute(str(CreateTable(table).compile(dialect=bind.dialect))) def create_table_if_missing(table_name: str, *columns, **kwargs) -> bool: """op.create_table, skipped when the table is already there. WHY EVERY POST-CUTOVER BASELINE NEEDS THIS. A baseline is the sole creator of its plugin's tables, so it was written as a plain op.create_table - which is correct only if the table's existence and the plugin's Alembic stamp can never disagree. They can, and routinely do: * `flask db-utils create-all` runs db.create_all(), which builds EVERY table registered on the metadata - plugin tables included - and stamps NOTHING. `flask plugin upgrade-all` then runs the baseline against tables that already exist. * A run that creates the table and then fails before its stamp commits leaves the same split, and re-running is the obvious thing to try. Reported as `flask plugin upgrade-all` failing the tools plugin with "table securesendsecrets already exists". Nothing caught it because the migration test drops the post-cutover tables db.create_all() made before running upgrade-all, which arranges the one state where an unguarded create works and never the state a real database is in. Returns True when it created the table, so a caller can skip the indexes that belong to it. """ from alembic import op from sqlalchemy import inspect if inspect(op.get_bind()).has_table(table_name): logger.info('%s already exists, skipping create', table_name) return False op.create_table(table_name, *columns, **kwargs) return True def create_index_if_missing(index_name: str, table_name: str, columns, **kwargs): """op.create_index, skipped when an equivalent index already exists. Matched on the COLUMN LIST rather than the name, because the two creators do not agree on names. db.create_all() takes them from SQLAlchemy's default convention (`ix_securesendsecrets_token`) while a baseline spells them the way this repo's naming rules require (`ixsecuresendsecretstoken`). Matching on the name would call the index missing and add a second one over the same column - MySQL permits that, with only a warning, so the duplicate would accumulate silently on every affected site. """ from alembic import op from sqlalchemy import inspect wanted = [str(column) for column in columns] inspector = inspect(op.get_bind()) if not inspector.has_table(table_name): op.create_index(index_name, table_name, columns, **kwargs) return for existing in inspector.get_indexes(table_name): if list(existing.get('column_names') or []) == wanted: logger.info('index over %s(%s) already exists as %s, skipping', table_name, ', '.join(wanted), existing.get('name')) return op.create_index(index_name, table_name, columns, **kwargs) def drop_plugin_tables(plugin_name: str): """Mirror of create_plugin_tables for downgrade(). Drops in reverse FK order.""" from alembic import op md = _get_plugin_metadata(plugin_name) for table in reversed(md.sorted_tables): op.execute(f'DROP TABLE IF EXISTS "{table.name}"') def run_migrations(): """Entry point called by each plugin's migrations/env.py.""" plugin_name = os.environ.get('PLUGIN_NAME') if not plugin_name: raise RuntimeError("PLUGIN_NAME env var must be set before run_migrations()") config = context.config target_metadata = _get_plugin_metadata(plugin_name) # Per-plugin version table so each plugin's chain is independent of core # Alembic's alembic_version table. version_table = f'alembic_version_{plugin_name}' db_url = config.get_main_option('sqlalchemy.url') if not db_url: # Pull from the Flask app config if running inside an app context # (e.g. via flask plugin migrate ). try: from flask import current_app db_url = current_app.config['SQLALCHEMY_DATABASE_URI'] config.set_main_option('sqlalchemy.url', db_url.replace('%', '%%')) except Exception as ex: raise RuntimeError( "sqlalchemy.url not set and no Flask app context available. " f"Original error: {ex}" ) if context.is_offline_mode(): context.configure( url=db_url, target_metadata=target_metadata, literal_binds=True, version_table=version_table, include_schemas=False, ) with context.begin_transaction(): context.run_migrations() else: # Build the engine straight from the resolved URL. The plugin manager # drives this via a programmatic alembic Config (no ini file), so # config.get_section returns an empty dict and engine_from_config would # find no sqlalchemy.url. db_url is already resolved above. from sqlalchemy import create_engine connectable = create_engine(db_url, poolclass=pool.NullPool) with connectable.connect() as connection: context.configure( connection=connection, target_metadata=target_metadata, version_table=version_table, include_schemas=False, ) with context.begin_transaction(): context.run_migrations() # SAME FIX AS migrations/env.py. MySQL DDL commits implicitly, which # flushes everything queued before it - including the PREVIOUS # migration's version stamp. The LAST migration of a run has no DDL # after it, so its stamp is rolled back when the connection closes: # the schema change survives, alembic_version stays one revision # behind, and `flask plugin upgrade-all` exits 0 having silently # re-run that migration - again on every deploy after. # # Core got this fix; the per-plugin chains run through THIS file and # did not, so every plugin head has been re-running on MySQL. That is # only invisible while every head happens to be idempotent. connection.commit()