From ecaecd21bbe1ef569da69d1dfd25389111ee8177 Mon Sep 17 00:00:00 2001 From: Pat Hartl Date: Sun, 14 Jun 2026 00:53:19 -0500 Subject: [PATCH] Send keepalive while game is running Sends a keepalive/heartbeat signal across RPC while a game is running. Server-side this enables the ability to manage play session state with realtime information instead of hoping that the client will stop the play session. Keepalives are tracked in server cache allowing for future potential horizontally-scaled server instances. --- LANCommander.SDK/Clients/GameClient.cs | 60 ++++++++++++++++++- .../Server/IRpcHub.GameKeepAlive.cs | 9 +++ .../IServiceCollectionExtensions.cs | 3 + .../PlaySessionKeepAliveTracker.cs | 50 ++++++++++++++++ .../PlaySessionService.cs | 47 +++++++++++++++ .../PlaySessionSweepService.cs | 49 +++++++++++++++ .../Models/GameServerSettings.cs | 7 +++ LANCommander.Server/Hubs/PlaySession.cs | 17 ++++++ LANCommander.Server/Hubs/_RpcHub.cs | 3 +- 9 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 LANCommander.SDK/Rpc/Interfaces/Server/IRpcHub.GameKeepAlive.cs create mode 100644 LANCommander.Server.Services/PlaySessionKeepAliveTracker.cs create mode 100644 LANCommander.Server.Services/PlaySessionSweepService.cs create mode 100644 LANCommander.Server/Hubs/PlaySession.cs diff --git a/LANCommander.SDK/Clients/GameClient.cs b/LANCommander.SDK/Clients/GameClient.cs index 8eacd779..d14d9d72 100644 --- a/LANCommander.SDK/Clients/GameClient.cs +++ b/LANCommander.SDK/Clients/GameClient.cs @@ -2271,11 +2271,15 @@ namespace LANCommander.SDK.Services } #endregion + Task heartbeatTask = null; + try { var cancellationTokenSource = new CancellationTokenSource(); _running[gameId] = cancellationTokenSource; + heartbeatTask = SendKeepAlivesAsync(gameId, cancellationTokenSource.Token); + #region Run Wrapper Scripts bool runWrapperHandled = false; @@ -2320,11 +2324,12 @@ namespace LANCommander.SDK.Services #endregion if (!runWrapperHandled) - { await context.ExecuteGameActionAsync(installDirectory, gameId, action, args, cancellationTokenSource.Token); - } _running.Remove(gameId); + + await StopHeartbeatAsync(cancellationTokenSource, heartbeatTask); + cancellationTokenSource.Dispose(); await UploadSavesAsync(manifests, installDirectory); @@ -2334,6 +2339,7 @@ namespace LANCommander.SDK.Services if (_running.TryGetValue(gameId, out var cts)) { _running.Remove(gameId); + await StopHeartbeatAsync(cts, heartbeatTask); cts.Dispose(); } logger?.LogError(ex, "Game failed to run"); @@ -2381,6 +2387,56 @@ namespace LANCommander.SDK.Services } } + // Heartbeat interval while a game is running. Must stay well below the server's + // KeepAliveTimeout so a session isn't reaped between beats. + private const int KeepAliveIntervalSeconds = 30; + + private async Task SendKeepAlivesAsync(Guid gameId, CancellationToken token) + { + try + { + while (!token.IsCancellationRequested) + { + await Task.Delay(TimeSpan.FromSeconds(KeepAliveIntervalSeconds), token); + + if (token.IsCancellationRequested) + break; + + if (!connectionClient.IsConnected() || RpcClient.Hub == null) + continue; + + try + { + await RpcClient.Hub.GameKeepAliveAsync(gameId); + } + catch (Exception ex) + { + logger?.LogTrace(ex, "Keepalive send failed for {GameId}", gameId); + } + } + } + catch (OperationCanceledException) + { + // Expected when the game exits and the token is cancelled. + } + } + + private static async Task StopHeartbeatAsync(CancellationTokenSource cancellationTokenSource, Task heartbeatTask) + { + cancellationTokenSource.Cancel(); + + if (heartbeatTask != null) + { + try + { + await heartbeatTask; + } + catch (OperationCanceledException) + { + } + } + } + public async Task Stop(Guid gameId) { if (_running.ContainsKey(gameId)) diff --git a/LANCommander.SDK/Rpc/Interfaces/Server/IRpcHub.GameKeepAlive.cs b/LANCommander.SDK/Rpc/Interfaces/Server/IRpcHub.GameKeepAlive.cs new file mode 100644 index 00000000..04b48fe7 --- /dev/null +++ b/LANCommander.SDK/Rpc/Interfaces/Server/IRpcHub.GameKeepAlive.cs @@ -0,0 +1,9 @@ +using System; +using System.Threading.Tasks; + +namespace LANCommander.SDK.Rpc.Server; + +public partial interface IRpcHub +{ + Task GameKeepAliveAsync(Guid gameId); +} diff --git a/LANCommander.Server.Services/Extensions/IServiceCollectionExtensions.cs b/LANCommander.Server.Services/Extensions/IServiceCollectionExtensions.cs index 76c3fbb9..d00dec53 100644 --- a/LANCommander.Server.Services/Extensions/IServiceCollectionExtensions.cs +++ b/LANCommander.Server.Services/Extensions/IServiceCollectionExtensions.cs @@ -98,6 +98,9 @@ public static class IServiceCollectionExtensions services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); diff --git a/LANCommander.Server.Services/PlaySessionKeepAliveTracker.cs b/LANCommander.Server.Services/PlaySessionKeepAliveTracker.cs new file mode 100644 index 00000000..56523b43 --- /dev/null +++ b/LANCommander.Server.Services/PlaySessionKeepAliveTracker.cs @@ -0,0 +1,50 @@ +using ZiggyCreatures.Caching.Fusion; + +namespace LANCommander.Server.Services; + +/// +/// Tracks the last time each active play session was heard from via the RPC keepalive. Backed by +/// so it works across instances when a distributed (e.g. Redis) backend +/// is configured. The sweep consults this to end sessions that have gone stale, ending them at the +/// last known-alive time so recorded playtime isn't inflated. +/// +public sealed class PlaySessionKeepAliveTracker( + IFusionCache cache, + SettingsProvider settingsProvider) +{ + private static string GetCacheKey(Guid gameId, Guid userId) => $"PlaySessions/KeepAlive/{gameId}/{userId}"; + + // Keep entries alive well past the staleness timeout so the sweep can still read the last + // keepalive time when ending a stale session, rather than losing it to expiration. + private TimeSpan EntryDuration => + TimeSpan.FromSeconds(Math.Max(1, settingsProvider.CurrentValue.Server.GameServers.KeepAliveTimeout) * 3); + + /// Records that the session for this game/user was heard from just now. + public async Task TouchAsync(Guid gameId, Guid userId) => + await cache.SetAsync(GetCacheKey(gameId, userId), DateTime.UtcNow, EntryDuration); + + /// + /// Returns the last-seen time for the session, seeding it with the current time if we have no + /// record yet (e.g. a session that predates a restart with no distributed backend). Seeding + /// grants a fresh grace window rather than immediately reaping a session we haven't observed yet. + /// + public async Task GetOrSeedAsync(Guid gameId, Guid userId) + { + var cacheKey = GetCacheKey(gameId, userId); + + var lastSeen = await cache.TryGetAsync(cacheKey); + + if (lastSeen.HasValue) + return lastSeen.Value; + + var now = DateTime.UtcNow; + + await cache.SetAsync(cacheKey, now, EntryDuration); + + return now; + } + + /// Stops tracking the session (it ended normally or was swept). + public async Task RemoveAsync(Guid gameId, Guid userId) => + await cache.RemoveAsync(GetCacheKey(gameId, userId)); +} diff --git a/LANCommander.Server.Services/PlaySessionService.cs b/LANCommander.Server.Services/PlaySessionService.cs index 4cabeee6..59735ad8 100644 --- a/LANCommander.Server.Services/PlaySessionService.cs +++ b/LANCommander.Server.Services/PlaySessionService.cs @@ -16,6 +16,7 @@ namespace LANCommander.Server.Services IMapper mapper, IHttpContextAccessor httpContextAccessor, IDbContextFactory contextFactory, + PlaySessionKeepAliveTracker keepAliveTracker, ServerService serverService) : BaseDatabaseService(logger, settingsProvider, cache, mapper, httpContextAccessor, contextFactory) { public override async Task AddAsync(PlaySession entity) @@ -51,6 +52,8 @@ namespace LANCommander.Server.Services }; await AddAsync(session); + + await keepAliveTracker.TouchAsync(gameId, userId); } public async Task EndSessionAsync(Guid gameId, Guid userId) @@ -63,6 +66,50 @@ namespace LANCommander.Server.Services await UpdateAsync(existingSession); } + + await keepAliveTracker.RemoveAsync(gameId, userId); + } + + /// + /// Records that the player is still in the game. Tracked via the cache; the sweep uses it to + /// detect sessions that have gone silent. + /// + public async Task KeepAliveAsync(Guid gameId, Guid userId) + { + await keepAliveTracker.TouchAsync(gameId, userId); + } + + /// + /// Ends any active session whose last keepalive is older than , + /// setting its End to that last known-alive time. Returns the affected games so callers can + /// schedule server autostop. + /// + public async Task> EndStaleSessionsAsync(TimeSpan timeout) + { + var cutoff = DateTime.UtcNow - timeout; + + var activeSessions = await GetAsync(ps => ps.End == null); + + var affectedGameIds = new HashSet(); + + foreach (var session in activeSessions) + { + var lastSeen = await keepAliveTracker.GetOrSeedAsync(session.GameId.GetValueOrDefault(), session.UserId); + + if (lastSeen >= cutoff) + continue; + + session.End = lastSeen; + + await UpdateAsync(session); + + await keepAliveTracker.RemoveAsync(session.GameId.GetValueOrDefault(), session.UserId); + + if (session.GameId.HasValue) + affectedGameIds.Add(session.GameId.Value); + } + + return affectedGameIds; } } } diff --git a/LANCommander.Server.Services/PlaySessionSweepService.cs b/LANCommander.Server.Services/PlaySessionSweepService.cs new file mode 100644 index 00000000..d0c907f2 --- /dev/null +++ b/LANCommander.Server.Services/PlaySessionSweepService.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace LANCommander.Server.Services; + +/// +/// Periodically ends play sessions that have stopped sending keepalives (e.g. the launcher crashed +/// or lost its connection), so the active-session list stays accurate for autostop and player +/// counts. Stale sessions are ended at their last known-alive time. Each affected game is handed to +/// so on-player-activity servers still autostop. +/// +public sealed class PlaySessionSweepService( + IServiceScopeFactory scopeFactory, + ServerManager serverManager, + SettingsProvider settingsProvider, + ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + var gameServers = settingsProvider.CurrentValue.Server.GameServers; + var interval = TimeSpan.FromSeconds(Math.Max(5, gameServers.KeepAliveSweepInterval)); + var timeout = TimeSpan.FromSeconds(Math.Max(1, gameServers.KeepAliveTimeout)); + + await Task.Delay(interval, stoppingToken); + + using var scope = scopeFactory.CreateScope(); + var playSessionService = scope.ServiceProvider.GetRequiredService(); + + var affectedGameIds = await playSessionService.EndStaleSessionsAsync(timeout); + + foreach (var gameId in affectedGameIds) + serverManager.ScheduleStop(gameId); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Play session sweep failed"); + } + } + } +} diff --git a/LANCommander.Server.Settings/Models/GameServerSettings.cs b/LANCommander.Server.Settings/Models/GameServerSettings.cs index b3a3d590..d2a09002 100644 --- a/LANCommander.Server.Settings/Models/GameServerSettings.cs +++ b/LANCommander.Server.Settings/Models/GameServerSettings.cs @@ -10,6 +10,13 @@ public class GameServerSettings // Seconds to wait after the last player stops a game before stopping its // on-player-activity servers. Debounces stop/start thrash when players relaunch. public int AutostopDelay { get; set; } = 300; + + // Seconds without a keepalive from the launcher before an active play session is considered + // stale and ended. Should comfortably exceed the launcher's keepalive interval. + public int KeepAliveTimeout { get; set; } = 120; + + // Seconds between sweeps that end stale play sessions. + public int KeepAliveSweepInterval { get; set; } = 30; public IEnumerable ServerEngines { get; set; } = [ new() diff --git a/LANCommander.Server/Hubs/PlaySession.cs b/LANCommander.Server/Hubs/PlaySession.cs new file mode 100644 index 00000000..283a1863 --- /dev/null +++ b/LANCommander.Server/Hubs/PlaySession.cs @@ -0,0 +1,17 @@ +namespace LANCommander.Server.Hubs; + +public partial class RpcHub +{ + public async Task GameKeepAliveAsync(Guid gameId) + { + try + { + if (Guid.TryParse(Context.UserIdentifier, out var userId)) + await keepAliveTracker.TouchAsync(gameId, userId); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to record keepalive for game {GameId}", gameId); + } + } +} diff --git a/LANCommander.Server/Hubs/_RpcHub.cs b/LANCommander.Server/Hubs/_RpcHub.cs index 92751de0..c47b7e81 100644 --- a/LANCommander.Server/Hubs/_RpcHub.cs +++ b/LANCommander.Server/Hubs/_RpcHub.cs @@ -16,7 +16,8 @@ public partial class RpcHub( ServerService serverService, GameService gameService, ScriptDebugger scriptDebugger, - ScriptClient scriptClient) : Hub, IRpcHub + ScriptClient scriptClient, + PlaySessionKeepAliveTracker keepAliveTracker) : Hub, IRpcHub { private string GetConnectionsCacheKey(string userIdentifier) => $"RPC/Connections/{userIdentifier}";