From bb301c6d84ae9bfc32453d9702c1c429f7869fd6 Mon Sep 17 00:00:00 2001 From: Proudlock <270015376@geaerospace.com> Date: Mon, 14 Sep 2026 14:33:36 -0400 Subject: [PATCH] Add periodic certificate expiry monitoring for long-running agents CRITICAL OPERATIONAL FIX: Agents running 90+ days without restart Problem: - Certificate renewal check only ran at service startup - Agents running continuously for 30+ days never renewed certificates - Certificates would expire at day 730 causing authentication failures - User confirmed agents routinely run 90+ days without restart Solution: - Added CertificateExpiryMonitor background service - Checks certificate expiry every 24 hours during runtime - Triggers renewal when < 30 days remaining - Gracefully stops service (Windows Service Manager restarts it) - New process picks up renewed certificate Implementation: - CertificateExpiryMonitor.cs: Background service (IHostedService) - Registered in Program.cs as hosted service - First check at 1 hour after startup, then every 24 hours - Uses same CertificateRenewal logic as startup check - Calls IHostApplicationLifetime.StopApplication() after renewal Timeline Example: - Day 0: Agent starts with 730-day cert - Day 1-89: Agent runs continuously (no restart) - Day 90: Periodic check runs, cert has 640 days left, continue - Day 700: Periodic check runs, cert has 30 days left, renews - Agent stops gracefully, service manager restarts - Day 701: Agent continues with new 730-day cert Without this fix: - Agent starts day 0, runs continuously - Day 730: Cert expires, no renewal check ever ran - Authentication fails at TLS handshake - Agent dead until manual intervention Co-Authored-By: Claude Sonnet 4.5 --- .../CertificateExpiryMonitor.cs | 86 +++++++++++++++++++ agent/ShopDbEndpointAgent/Program.cs | 6 ++ 2 files changed, 92 insertions(+) create mode 100644 agent/ShopDbEndpointAgent.Core/CertificateExpiryMonitor.cs diff --git a/agent/ShopDbEndpointAgent.Core/CertificateExpiryMonitor.cs b/agent/ShopDbEndpointAgent.Core/CertificateExpiryMonitor.cs new file mode 100644 index 00000000..0bfca037 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CertificateExpiryMonitor.cs @@ -0,0 +1,86 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +/// +/// Background service that periodically checks client certificate expiry. +/// If certificate expires within 30 days, triggers renewal and service restart. +/// +/// This is critical for long-running agents that don't restart frequently. +/// Without this, an agent running > 30 days will have an expired cert with no +/// automatic renewal, causing authentication failures. +/// +public class CertificateExpiryMonitor : BackgroundService +{ + private readonly ILogger logger; + private readonly ILogger renewalLogger; + private readonly AgentConfig config; + private readonly IHostApplicationLifetime lifetime; + + private const int CHECK_INTERVAL_HOURS = 24; + private const int RENEWAL_THRESHOLD_DAYS = 30; + + public CertificateExpiryMonitor( + ILogger logger, + ILogger renewalLogger, + AgentConfig config, + IHostApplicationLifetime lifetime) + { + this.logger = logger; + this.renewalLogger = renewalLogger; + this.config = config; + this.lifetime = lifetime; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Wait 1 hour after service start before first check (let agent stabilize) + await Task.Delay(TimeSpan.FromHours(1), stoppingToken); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await CheckCertificateExpiry(); + } + catch (Exception ex) + { + logger.LogError(ex, "Certificate expiry check failed"); + } + + // Check every 24 hours + await Task.Delay(TimeSpan.FromHours(CHECK_INTERVAL_HOURS), stoppingToken); + } + } + + private async Task CheckCertificateExpiry() + { + if (string.IsNullOrWhiteSpace(config.ClientCertThumbprint)) + { + // No client cert configured, nothing to check + return; + } + + logger.LogDebug("Performing periodic certificate expiry check"); + + // Use same renewal logic as startup check + var renewal = new CertificateRenewal(renewalLogger, config); + bool didRenew = await renewal.CheckAndRenewIfNeeded(); + + if (didRenew) + { + logger.LogWarning( + "Certificate was renewed during periodic check. Agent has been running " + + "for an extended period without restart. Stopping service to apply new " + + "certificate. Service manager will restart the agent."); + + // Stop the application gracefully + // Windows Service Manager will restart it (if configured as Automatic) + lifetime.StopApplication(); + } + } +} diff --git a/agent/ShopDbEndpointAgent/Program.cs b/agent/ShopDbEndpointAgent/Program.cs index 8e0ca23a..00409093 100644 --- a/agent/ShopDbEndpointAgent/Program.cs +++ b/agent/ShopDbEndpointAgent/Program.cs @@ -214,6 +214,12 @@ static async Task RunHostAsync(AgentConfig config, string agentVersion, builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddHostedService(sp => sp.GetRequiredService()); + // Background monitor for certificate expiry (critical for long-running agents) + // Checks every 24 hours, triggers renewal if < 30 days remaining + // Without this, agents running > 30 days would have expired certs + builder.Services.AddSingleton(); + builder.Services.AddHostedService(sp => sp.GetRequiredService()); + await builder.Build().RunAsync(); }