diff --git a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs index 92320022..e8f11fea 100644 --- a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs +++ b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs @@ -17,6 +17,23 @@ public sealed class AgentConfig // (thundering herd). Actual delay = cadence +/- up to jitterseconds. [JsonPropertyName("jitterseconds")] public int JitterSeconds { get; set; } = 60; + // One-time random delay before FIRST contact, applied by both the telemetry + // and command loops at startup. JitterSeconds only spreads the gap BETWEEN + // runs, so without this every agent in the fleet still makes its first + // request the instant its service starts - and the service is installed + // start=auto, so a patch reboot starts them all together. + // + // Measured on a 700-endpoint burst (devnotes/gevent-sizing-simulation.md): + // simultaneous arrival drove TLS handshakes to 1482ms worst case and pinned + // CPU at 305% while the database sat idle at 3 active queries. Spreading the + // same arrivals over 30s cut worst-case handshake to 63.7ms. The server is + // not the constraint; simultaneity is. + // + // Costs up to this many seconds before a rebooted endpoint picks up a queued + // command. That window is on top of a reboot the machine was already + // unreachable for. 0 disables the delay. + [JsonPropertyName("startupjitterseconds")] public int StartupJitterSeconds { get; set; } = 30; + [JsonPropertyName("adoptonfirstrun")] public bool AdoptOnFirstRun { get; set; } = true; // Directory holding site-provided PowerShell "agent utilities" the runtime diff --git a/agent/ShopDbEndpointAgent.Core/CommandWorker.cs b/agent/ShopDbEndpointAgent.Core/CommandWorker.cs index 2d71232d..5327f8e8 100644 --- a/agent/ShopDbEndpointAgent.Core/CommandWorker.cs +++ b/agent/ShopDbEndpointAgent.Core/CommandWorker.cs @@ -41,15 +41,18 @@ public sealed class CommandWorker : BackgroundService private readonly CommandExecutor executor; private readonly ILogger logger; private readonly CollectSignal? collectSignal; + private readonly AgentConfig? config; private readonly string hostname; public CommandWorker(CommandClient client, CommandExecutor executor, - ILogger logger, CollectSignal? collectSignal = null) + ILogger logger, CollectSignal? collectSignal = null, + AgentConfig? config = null) { this.client = client; this.executor = executor; this.logger = logger; this.collectSignal = collectSignal; + this.config = config; hostname = Environment.MachineName; } @@ -57,6 +60,18 @@ public sealed class CommandWorker : BackgroundService { logger.LogInformation("Command channel started for host {Host}", hostname); + // Spread first contact across the fleet. Every agent otherwise polls the + // instant its service starts, and the service is start=auto, so a patch + // reboot puts the whole fleet on the wire together. See StartupJitter. + var startupDelay = StartupJitter.Next(config?.StartupJitterSeconds ?? 0, logger); + if (startupDelay > TimeSpan.Zero) + { + logger.LogInformation( + "Delaying first command poll by {Delay}ms (startup jitter)", + (int)startupDelay.TotalMilliseconds); + await SafeDelay(startupDelay, stoppingToken); + } + while (!stoppingToken.IsCancellationRequested) { var pollStarted = System.Diagnostics.Stopwatch.StartNew(); diff --git a/agent/ShopDbEndpointAgent.Core/StartupJitter.cs b/agent/ShopDbEndpointAgent.Core/StartupJitter.cs new file mode 100644 index 00000000..14e4157a --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/StartupJitter.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// One-time startup delay, shared by TelemetryWorker and CommandWorker. +// +// Both loops otherwise make their first request the instant the service starts, +// and the service is installed start=auto - so a patch-day reboot has an entire +// fleet arriving together, twice per machine (a telemetry report AND a command +// poll). AgentConfig.JitterSeconds does not help: it spreads the gap BETWEEN +// runs, not the first contact. +// +// Lives here rather than in either worker so the two cannot drift apart, and so +// the clamp is enforced in one place. +internal static class StartupJitter +{ + // Ceiling on the configured value. A fat-fingered appsettings.json must not + // be able to park an endpoint for hours before it first checks in: the + // command channel is how an operator would correct it, so an unbounded + // delay would take away the means of recovery. + private const int MaxStartupJitterSeconds = 300; + + public static TimeSpan Next(int configuredSeconds, ILogger logger) + { + // Warn rather than clamp silently: an operator who deliberately set a + // large value and got a smaller one has config and behaviour disagreeing + // with no way to tell why. A negative value is a mistake too, so it is + // worth the same line. + if (configuredSeconds > MaxStartupJitterSeconds) + { + logger.LogWarning( + "startupjitterseconds {Configured} exceeds the {Max}s ceiling; using {Max}s", + configuredSeconds, MaxStartupJitterSeconds, MaxStartupJitterSeconds); + } + else if (configuredSeconds < 0) + { + logger.LogWarning( + "startupjitterseconds {Configured} is negative; startup jitter disabled", + configuredSeconds); + } + + int spread = Math.Clamp(configuredSeconds, 0, MaxStartupJitterSeconds); + if (spread == 0) + return TimeSpan.Zero; + + // Milliseconds, not whole seconds: 700 endpoints over a 30s window would + // otherwise land in 30 clumps of ~23 simultaneous arrivals, which is the + // same pile-up in miniature. + return TimeSpan.FromMilliseconds(Random.Shared.Next(0, spread * 1000)); + } +} diff --git a/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs b/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs index 97b94bb4..bf689192 100644 --- a/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs +++ b/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs @@ -139,6 +139,25 @@ public sealed class TelemetryWorker : BackgroundService "Endpoint agent {Version} -> {BaseUrl}, cadence {Cadence}s +/-{Jitter}s", agentVersion, config.BaseUrl, config.CadenceSeconds, config.JitterSeconds); + // Spread first contact across the fleet. JitterSeconds only spreads the + // gap BETWEEN runs, so without this the whole fleet reports the instant + // its service starts. See StartupJitter. + var startupDelay = StartupJitter.Next(config.StartupJitterSeconds, logger); + if (startupDelay > TimeSpan.Zero) + { + logger.LogInformation( + "Delaying first telemetry report by {Delay}ms (startup jitter)", + (int)startupDelay.TotalMilliseconds); + try + { + await Task.Delay(startupDelay, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + } + bool firstRun = true; while (!stoppingToken.IsCancellationRequested) { diff --git a/agent/ShopDbEndpointAgent/Program.cs b/agent/ShopDbEndpointAgent/Program.cs index ffa770bd..1e041166 100644 --- a/agent/ShopDbEndpointAgent/Program.cs +++ b/agent/ShopDbEndpointAgent/Program.cs @@ -346,7 +346,8 @@ static void RegisterCoreServices(IServiceCollection services, AgentConfig config sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService>(), - sp.GetRequiredService())); + sp.GetRequiredService(), + config)); } static void ConfigureLogging(ILoggingBuilder logging, string logDirectory) diff --git a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj index ec456c3b..e3ca12f1 100644 --- a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj +++ b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj @@ -131,6 +131,27 @@ (AgentConfig.LaunchPaths, shipped defaults overridable per site). 1.1.0: generic, data-driven service_*/process_* actions carry their target over the wire, so new policies/apps need no agent rebuild. --> + - 1.11.0 + 1.12.0 true win-x64 true diff --git a/agent/ShopDbEndpointAgent/appsettings.json b/agent/ShopDbEndpointAgent/appsettings.json index 6db6ca67..78479ed8 100644 --- a/agent/ShopDbEndpointAgent/appsettings.json +++ b/agent/ShopDbEndpointAgent/appsettings.json @@ -3,6 +3,7 @@ "apikey": "REPLACE_WITH_COLLECTOR_KEY", "cadenceseconds": 900, "jitterseconds": 60, + "startupjitterseconds": 30, "adoptonfirstrun": true, "utilitiespath": "", "logpath": "", diff --git a/plugins/endpoints/api/siteagent.py b/plugins/endpoints/api/siteagent.py index 56554537..c5d7fd28 100644 --- a/plugins/endpoints/api/siteagent.py +++ b/plugins/endpoints/api/siteagent.py @@ -118,6 +118,7 @@ _BASE_APPSETTINGS = { 'apikey': 'REPLACE_WITH_COLLECTOR_KEY', 'cadenceseconds': 900, 'jitterseconds': 60, + 'startupjitterseconds': 30, 'adoptonfirstrun': True, 'utilitiespath': '', 'logpath': '',