Self-heal endpoint command queue wedged by a dead in-flight command

The command queue is at-most-one-in-flight, but nothing ever expired a stuck command. When an agent claimed a command (RUNNING) then died before reporting a result - killed, uninstalled, or offline mid-flight - the row stayed in-flight forever and every operator action for that endpoint returned 409 'already in flight'. Reap in-flight rows past a staleness ceiling (15m for RUNNING, 60m for PENDING) to a new STATUS_EXPIRED at enqueue time, inside the same transaction, so the queue self-heals lazily the next time someone queues work. No background job and no schema migration - status is free-text.

tests/test_endpoint_command_reaper.py

@@ -0,0 +1,69 @@
+"""Regression tests for the endpoint command-queue staleness reaper.
+
+A claimed command whose agent never reports back (agent killed, uninstalled, or
+offline mid-flight) would otherwise pin the at-most-one-in-flight queue forever,
+so every operator action against that endpoint returns 409 "already in flight".
+The enqueue path reaps such rows to STATUS_EXPIRED so the queue self-heals.
+"""
+
+from datetime import timedelta
+
+
+def test_reap_stale_commands_frees_a_wedged_queue(db):
+    from plugins.endpoints.models import EndpointCommand
+    from plugins.endpoints.api.routes import _reap_stale_commands, _naive_utcnow
+
+    now = _naive_utcnow()
+    asset_id = 4242
+
+    stale_running = EndpointCommand(
+        assetid=asset_id, action='mtconnect_start',
+        status=EndpointCommand.STATUS_RUNNING,
+        createdat=now - timedelta(hours=2),
+        claimedat=now - timedelta(hours=2))
+    stale_pending = EndpointCommand(
+        assetid=asset_id, action='mtconnect_stop',
+        status=EndpointCommand.STATUS_PENDING,
+        createdat=now - timedelta(hours=2))
+    fresh_running = EndpointCommand(
+        assetid=asset_id + 1, action='mtconnect_start',
+        status=EndpointCommand.STATUS_RUNNING,
+        createdat=now, claimedat=now)
+    db.session.add_all([stale_running, stale_pending, fresh_running])
+    db.session.commit()
+
+    reaped = _reap_stale_commands(asset_id)
+    db.session.commit()
+
+    reaped_ids = {command.endpointcommandid for command in reaped}
+    assert stale_running.endpointcommandid in reaped_ids
+    assert stale_pending.endpointcommandid in reaped_ids
+    assert stale_running.status == EndpointCommand.STATUS_EXPIRED
+    assert stale_pending.status == EndpointCommand.STATUS_EXPIRED
+    assert stale_running.completedat is not None
+    assert stale_running.resultmessage
+    # A live, recently claimed command on another endpoint is untouched.
+    assert fresh_running.status == EndpointCommand.STATUS_RUNNING
+
+
+def test_reap_leaves_a_recently_claimed_running_command_in_flight(db):
+    from plugins.endpoints.models import EndpointCommand
+    from plugins.endpoints.api.routes import _reap_stale_commands, _naive_utcnow
+
+    now = _naive_utcnow()
+    asset_id = 777
+
+    # Claimed a minute ago: a slow-but-alive command must not be reaped.
+    running = EndpointCommand(
+        assetid=asset_id, action='mtconnect_start',
+        status=EndpointCommand.STATUS_RUNNING,
+        createdat=now - timedelta(minutes=1),
+        claimedat=now - timedelta(minutes=1))
+    db.session.add(running)
+    db.session.commit()
+
+    reaped = _reap_stale_commands(asset_id)
+    db.session.commit()
+
+    assert reaped == []
+    assert running.status == EndpointCommand.STATUS_RUNNING