From 0eed80f3b9482c1779c62498ab29309775862a29 Mon Sep 17 00:00:00 2001 From: cproudlock Date: Sat, 19 Sep 2026 17:42:04 -0400 Subject: [PATCH] Create endpointlogcollections in the chain that owns it A fresh install could not build its schema. Core revision 7d51_endpoint_logcollections creates endpointlogcollections with a foreign key onto endpointcommands, but PLUGIN_TABLE_OWNERS lists both tables as owned by the endpoints plugin, and only the plugin's 0002 revision creates endpointcommands. Core migrations run before any plugin chain, and a plugin chain only runs once that plugin is enabled, which cannot happen before the schema exists. So there is no ordering in which the core revision can succeed on an empty database. MySQL rejects it with 1824, 'Failed to open the referenced table endpointcommands'. That is the first command the README gives an adopting site, on the branch that exists for sites to adopt. Nothing caught it because the suite runs on sqlite:///:memory: and SQLite accepts a foreign key onto a table that does not exist. The failure is MySQL-only and therefore invisible to every test. Fix: - endpoints revision 0025 creates the table in the plugin chain, after 0002 has created its foreign key target. Guarded, so it is a no-op on sites that already applied the core revision. - 7d51 skips when endpointcommands is absent, which is exactly the fresh install case 0025 then covers. Its downgrade is guarded to match. - EXPECTED_HEAD_REVISION records the new endpoints head. Verified against real MySQL 8.4, not SQLite: on an empty database the README sequence now completes, 7d51 stamps without creating anything, and enabling the endpoints plugin builds endpointlogcollections with endpointlogcollections_ibfk_2 pointing at endpointcommands. --- .../versions/7d51_endpoint_logcollections.py | 18 ++++ .../versions/0025_endpoint_logcollections.py | 101 ++++++++++++++++++ tests/test_plugin_migrations.py | 2 +- 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 plugins/endpoints/migrations/versions/0025_endpoint_logcollections.py diff --git a/migrations/versions/7d51_endpoint_logcollections.py b/migrations/versions/7d51_endpoint_logcollections.py index d80fa9b7..e1488423 100644 --- a/migrations/versions/7d51_endpoint_logcollections.py +++ b/migrations/versions/7d51_endpoint_logcollections.py @@ -39,6 +39,19 @@ depends_on = None def upgrade(): + # GUARD: this table belongs to the endpoints PLUGIN, not to core. + # PLUGIN_TABLE_OWNERS has always said so, and the foreign key below points + # at endpointcommands, which only the plugin's 0002 revision creates. Core + # runs first and a plugin chain only runs once the plugin is enabled, so on + # a fresh database endpointcommands does not exist yet and MySQL rejects + # the create with 1824. Skip here and let endpoints0025logcollections build + # it in the chain that owns it. + # + # Sites that already applied this revision have the table, so they are + # unaffected either way: the plugin revision is guarded and no-ops there. + if not sa.inspect(op.get_bind()).has_table('endpointcommands'): + return + op.create_table( 'endpointlogcollections', sa.Column('endpointlogcollectionid', sa.Integer(), nullable=False), @@ -83,6 +96,11 @@ def upgrade(): def downgrade(): + # Mirror of the upgrade guard: on a fresh install this revision created + # nothing, so there is nothing here to drop. + if not sa.inspect(op.get_bind()).has_table('endpointlogcollections'): + return + op.drop_index('idx_endpointlogcollections_assetcollected', table_name='endpointlogcollections') op.drop_index('ix_endpointlogcollections_collectedat', diff --git a/plugins/endpoints/migrations/versions/0025_endpoint_logcollections.py b/plugins/endpoints/migrations/versions/0025_endpoint_logcollections.py new file mode 100644 index 00000000..abfffa07 --- /dev/null +++ b/plugins/endpoints/migrations/versions/0025_endpoint_logcollections.py @@ -0,0 +1,101 @@ +"""Move endpointlogcollections into the chain that owns it. + +Revision ID: endpoints0025logcollections +Revises: endpoints0024bootstraptokens +Create Date: 2026-09-19 + +WHY THIS REVISION EXISTS. endpointlogcollections was created by a CORE +migration (7d51_endpoint_logcollections) even though PLUGIN_TABLE_OWNERS has +always listed it, and endpointcommands, as owned by this plugin. The core table +carries a foreign key onto endpointcommands, which only 0002 creates. Core runs +before any plugin chain, and a plugin chain only runs once the plugin is +enabled, which cannot happen before the schema exists. So on a FRESH database +there is no ordering in which the core revision can succeed: MySQL rejects it +with 1824, "Failed to open the referenced table 'endpointcommands'". + +Nothing caught it because TestingConfig runs on sqlite:///:memory:, and SQLite +accepts a foreign key onto a table that does not exist. The failure is +MySQL-only, and therefore invisible to the whole test suite. + +Existing sites already ran the core revision and have the table, so this is +guarded and is a no-op there. 7d51 is guarded in the same change so it skips +when endpointcommands is absent, which is exactly the fresh-install case this +revision then handles. + +Downgrade drops the table. +""" +from alembic import op + +from shopdb.plugins.alembic_template import ( + create_index_if_missing, + create_table_if_missing, +) + +import sqlalchemy as sa + + +revision = 'endpoints0025logcollections' +down_revision = 'endpoints0024bootstraptokens' +branch_labels = None +depends_on = None +plugin_name = 'endpoints' + +COLLECTIONS = 'endpointlogcollections' + + +def upgrade(): + created = create_table_if_missing( + COLLECTIONS, + sa.Column('endpointlogcollectionid', sa.Integer(), nullable=False), + sa.Column('assetid', sa.Integer(), nullable=False), + sa.Column('endpointcommandid', sa.Integer(), nullable=True), + sa.Column('sourcepath', sa.String(length=512), nullable=False, + comment='Path as the AGENT resolved it, which may differ ' + 'from what the operator typed once environment ' + 'variables are expanded endpoint-side.'), + sa.Column('content', sa.Text(length=16777215), nullable=True), + sa.Column('filesizebytes', sa.BigInteger(), nullable=True, + comment='Size of the file on disk. Differs from contentbytes ' + 'when the stored copy is a truncated tail.'), + sa.Column('contentbytes', sa.Integer(), nullable=True), + sa.Column('linecount', sa.Integer(), nullable=True), + sa.Column('istruncated', sa.Boolean(), nullable=False, + server_default=sa.false(), + comment='True when content is only the tail of a larger file.'), + sa.Column('filemodifiedat', sa.DateTime(), nullable=True), + sa.Column('collectedbyuserid', sa.Integer(), nullable=True), + sa.Column('collectedbyusername', sa.String(length=100), nullable=True, + comment='Username snapshot; the id alone becomes unreadable ' + 'if the user is later removed.'), + sa.Column('collectedat', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['assetid'], ['assets.assetid'], + ondelete='CASCADE'), + # SET NULL rather than CASCADE: commands are reaped and expired over + # time, and a collected log an operator is still reading must not + # vanish because its originating command row was cleaned up. + sa.ForeignKeyConstraint(['endpointcommandid'], + ['endpointcommands.endpointcommandid'], + ondelete='SET NULL'), + sa.ForeignKeyConstraint(['collectedbyuserid'], ['users.userid'], + ondelete='SET NULL'), + sa.PrimaryKeyConstraint('endpointlogcollectionid'), + ) + if not created: + return + + create_index_if_missing('ix_endpointlogcollections_assetid', + COLLECTIONS, ['assetid']) + create_index_if_missing('ix_endpointlogcollections_collectedat', + COLLECTIONS, ['collectedat']) + # Serves both hot paths: listing an asset's history newest-first, and + # finding that asset's oldest rows to prune on insert. + create_index_if_missing('idx_endpointlogcollections_assetcollected', + COLLECTIONS, ['assetid', 'collectedat']) + + +def downgrade(): + from sqlalchemy import inspect + + if not inspect(op.get_bind()).has_table(COLLECTIONS): + return + op.drop_table(COLLECTIONS) diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index c9de7065..fca1d85b 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -123,7 +123,7 @@ EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof' # endpointcredentials, endpointenrollrequests and endpointmigrationwaves. # 0024 adds endpointbootstraptokens: the short-lived, admin-minted tokens behind # the one-line bootstrap installer. -EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0024bootstraptokens' +EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0025logcollections' # Plugins built after the cutover: their 0001 baseline really creates tables the # core chain never owned.