Add periodic certificate expiry monitoring for long-running agents

CRITICAL OPERATIONAL FIX: Agents running 90+ days without restart

Problem:
- Certificate renewal check only ran at service startup
- Agents running continuously for 30+ days never renewed certificates
- Certificates would expire at day 730 causing authentication failures
- User confirmed agents routinely run 90+ days without restart

Solution:
- Added CertificateExpiryMonitor background service
- Checks certificate expiry every 24 hours during runtime
- Triggers renewal when < 30 days remaining
- Gracefully stops service (Windows Service Manager restarts it)
- New process picks up renewed certificate

Implementation:
- CertificateExpiryMonitor.cs: Background service (IHostedService)
- Registered in Program.cs as hosted service
- First check at 1 hour after startup, then every 24 hours
- Uses same CertificateRenewal logic as startup check
- Calls IHostApplicationLifetime.StopApplication() after renewal

Timeline Example:
- Day 0: Agent starts with 730-day cert
- Day 1-89: Agent runs continuously (no restart)
- Day 90: Periodic check runs, cert has 640 days left, continue
- Day 700: Periodic check runs, cert has 30 days left, renews
- Agent stops gracefully, service manager restarts
- Day 701: Agent continues with new 730-day cert

Without this fix:
- Agent starts day 0, runs continuously
- Day 730: Cert expires, no renewal check ever ran
- Authentication fails at TLS handshake
- Agent dead until manual intervention

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
2026-09-14 14:33:36 -04:00
parent 8d87921dee
commit bb301c6d84
2 changed files with 92 additions and 0 deletions

View File

@@ -0,0 +1,86 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ShopDbEndpointAgent.Core;
/// <summary>
/// Background service that periodically checks client certificate expiry.
/// If certificate expires within 30 days, triggers renewal and service restart.
///
/// This is critical for long-running agents that don't restart frequently.
/// Without this, an agent running > 30 days will have an expired cert with no
/// automatic renewal, causing authentication failures.
/// </summary>
public class CertificateExpiryMonitor : BackgroundService
{
private readonly ILogger<CertificateExpiryMonitor> logger;
private readonly ILogger<CertificateRenewal> renewalLogger;
private readonly AgentConfig config;
private readonly IHostApplicationLifetime lifetime;
private const int CHECK_INTERVAL_HOURS = 24;
private const int RENEWAL_THRESHOLD_DAYS = 30;
public CertificateExpiryMonitor(
ILogger<CertificateExpiryMonitor> logger,
ILogger<CertificateRenewal> renewalLogger,
AgentConfig config,
IHostApplicationLifetime lifetime)
{
this.logger = logger;
this.renewalLogger = renewalLogger;
this.config = config;
this.lifetime = lifetime;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// Wait 1 hour after service start before first check (let agent stabilize)
await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
try
{
await CheckCertificateExpiry();
}
catch (Exception ex)
{
logger.LogError(ex, "Certificate expiry check failed");
}
// Check every 24 hours
await Task.Delay(TimeSpan.FromHours(CHECK_INTERVAL_HOURS), stoppingToken);
}
}
private async Task CheckCertificateExpiry()
{
if (string.IsNullOrWhiteSpace(config.ClientCertThumbprint))
{
// No client cert configured, nothing to check
return;
}
logger.LogDebug("Performing periodic certificate expiry check");
// Use same renewal logic as startup check
var renewal = new CertificateRenewal(renewalLogger, config);
bool didRenew = await renewal.CheckAndRenewIfNeeded();
if (didRenew)
{
logger.LogWarning(
"Certificate was renewed during periodic check. Agent has been running " +
"for an extended period without restart. Stopping service to apply new " +
"certificate. Service manager will restart the agent.");
// Stop the application gracefully
// Windows Service Manager will restart it (if configured as Automatic)
lifetime.StopApplication();
}
}
}

View File

@@ -214,6 +214,12 @@ static async Task RunHostAsync(AgentConfig config, string agentVersion,
builder.Services.AddHostedService(sp => sp.GetRequiredService<TelemetryWorker>());
builder.Services.AddHostedService(sp => sp.GetRequiredService<CommandWorker>());
// Background monitor for certificate expiry (critical for long-running agents)
// Checks every 24 hours, triggers renewal if < 30 days remaining
// Without this, agents running > 30 days would have expired certs
builder.Services.AddSingleton<CertificateExpiryMonitor>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<CertificateExpiryMonitor>());
await builder.Build().RunAsync();
}