Add periodic certificate expiry monitoring for long-running agents
CRITICAL OPERATIONAL FIX: Agents running 90+ days without restart Problem: - Certificate renewal check only ran at service startup - Agents running continuously for 30+ days never renewed certificates - Certificates would expire at day 730 causing authentication failures - User confirmed agents routinely run 90+ days without restart Solution: - Added CertificateExpiryMonitor background service - Checks certificate expiry every 24 hours during runtime - Triggers renewal when < 30 days remaining - Gracefully stops service (Windows Service Manager restarts it) - New process picks up renewed certificate Implementation: - CertificateExpiryMonitor.cs: Background service (IHostedService) - Registered in Program.cs as hosted service - First check at 1 hour after startup, then every 24 hours - Uses same CertificateRenewal logic as startup check - Calls IHostApplicationLifetime.StopApplication() after renewal Timeline Example: - Day 0: Agent starts with 730-day cert - Day 1-89: Agent runs continuously (no restart) - Day 90: Periodic check runs, cert has 640 days left, continue - Day 700: Periodic check runs, cert has 30 days left, renews - Agent stops gracefully, service manager restarts - Day 701: Agent continues with new 730-day cert Without this fix: - Agent starts day 0, runs continuously - Day 730: Cert expires, no renewal check ever ran - Authentication fails at TLS handshake - Agent dead until manual intervention Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
86
agent/ShopDbEndpointAgent.Core/CertificateExpiryMonitor.cs
Normal file
86
agent/ShopDbEndpointAgent.Core/CertificateExpiryMonitor.cs
Normal file
@@ -0,0 +1,86 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ShopDbEndpointAgent.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Background service that periodically checks client certificate expiry.
|
||||
/// If certificate expires within 30 days, triggers renewal and service restart.
|
||||
///
|
||||
/// This is critical for long-running agents that don't restart frequently.
|
||||
/// Without this, an agent running > 30 days will have an expired cert with no
|
||||
/// automatic renewal, causing authentication failures.
|
||||
/// </summary>
|
||||
public class CertificateExpiryMonitor : BackgroundService
|
||||
{
|
||||
private readonly ILogger<CertificateExpiryMonitor> logger;
|
||||
private readonly ILogger<CertificateRenewal> renewalLogger;
|
||||
private readonly AgentConfig config;
|
||||
private readonly IHostApplicationLifetime lifetime;
|
||||
|
||||
private const int CHECK_INTERVAL_HOURS = 24;
|
||||
private const int RENEWAL_THRESHOLD_DAYS = 30;
|
||||
|
||||
public CertificateExpiryMonitor(
|
||||
ILogger<CertificateExpiryMonitor> logger,
|
||||
ILogger<CertificateRenewal> renewalLogger,
|
||||
AgentConfig config,
|
||||
IHostApplicationLifetime lifetime)
|
||||
{
|
||||
this.logger = logger;
|
||||
this.renewalLogger = renewalLogger;
|
||||
this.config = config;
|
||||
this.lifetime = lifetime;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
// Wait 1 hour after service start before first check (let agent stabilize)
|
||||
await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await CheckCertificateExpiry();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Certificate expiry check failed");
|
||||
}
|
||||
|
||||
// Check every 24 hours
|
||||
await Task.Delay(TimeSpan.FromHours(CHECK_INTERVAL_HOURS), stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CheckCertificateExpiry()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(config.ClientCertThumbprint))
|
||||
{
|
||||
// No client cert configured, nothing to check
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogDebug("Performing periodic certificate expiry check");
|
||||
|
||||
// Use same renewal logic as startup check
|
||||
var renewal = new CertificateRenewal(renewalLogger, config);
|
||||
bool didRenew = await renewal.CheckAndRenewIfNeeded();
|
||||
|
||||
if (didRenew)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Certificate was renewed during periodic check. Agent has been running " +
|
||||
"for an extended period without restart. Stopping service to apply new " +
|
||||
"certificate. Service manager will restart the agent.");
|
||||
|
||||
// Stop the application gracefully
|
||||
// Windows Service Manager will restart it (if configured as Automatic)
|
||||
lifetime.StopApplication();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -214,6 +214,12 @@ static async Task RunHostAsync(AgentConfig config, string agentVersion,
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<TelemetryWorker>());
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<CommandWorker>());
|
||||
|
||||
// Background monitor for certificate expiry (critical for long-running agents)
|
||||
// Checks every 24 hours, triggers renewal if < 30 days remaining
|
||||
// Without this, agents running > 30 days would have expired certs
|
||||
builder.Services.AddSingleton<CertificateExpiryMonitor>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<CertificateExpiryMonitor>());
|
||||
|
||||
await builder.Build().RunAsync();
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user