Send keepalive while game is running

Sends a keepalive/heartbeat signal across RPC while a game is running. Server-side this enables the ability to manage play session state with realtime information instead of hoping that the client will stop the play session. Keepalives are tracked in server cache allowing for future potential horizontally-scaled server instances.

LANCommander.SDK/Clients/GameClient.cs

@@ -2271,11 +2271,15 @@ namespace LANCommander.SDK.Services
                 }
                 #endregion
 
+                Task heartbeatTask = null;
+
                 try
                 {
                     var cancellationTokenSource = new CancellationTokenSource();
                     _running[gameId] = cancellationTokenSource;
 
+                    heartbeatTask = SendKeepAlivesAsync(gameId, cancellationTokenSource.Token);
+
                     #region Run Wrapper Scripts
                     bool runWrapperHandled = false;
 
@@ -2320,11 +2324,12 @@ namespace LANCommander.SDK.Services
                     #endregion
 
                     if (!runWrapperHandled)
-                    {
                         await context.ExecuteGameActionAsync(installDirectory, gameId, action, args, cancellationTokenSource.Token);
-                    }
 
                     _running.Remove(gameId);
+                    
+                    await StopHeartbeatAsync(cancellationTokenSource, heartbeatTask);
+                    
                     cancellationTokenSource.Dispose();
 
                     await UploadSavesAsync(manifests, installDirectory);
@@ -2334,6 +2339,7 @@ namespace LANCommander.SDK.Services
                     if (_running.TryGetValue(gameId, out var cts))
                     {
                         _running.Remove(gameId);
+                        await StopHeartbeatAsync(cts, heartbeatTask);
                         cts.Dispose();
                     }
                     logger?.LogError(ex, "Game failed to run");
@@ -2381,6 +2387,56 @@ namespace LANCommander.SDK.Services
             }
         }
 
+        // Heartbeat interval while a game is running. Must stay well below the server's
+        // KeepAliveTimeout so a session isn't reaped between beats.
+        private const int KeepAliveIntervalSeconds = 30;
+
+        private async Task SendKeepAlivesAsync(Guid gameId, CancellationToken token)
+        {
+            try
+            {
+                while (!token.IsCancellationRequested)
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(KeepAliveIntervalSeconds), token);
+
+                    if (token.IsCancellationRequested)
+                        break;
+
+                    if (!connectionClient.IsConnected() || RpcClient.Hub == null)
+                        continue;
+
+                    try
+                    {
+                        await RpcClient.Hub.GameKeepAliveAsync(gameId);
+                    }
+                    catch (Exception ex)
+                    {
+                        logger?.LogTrace(ex, "Keepalive send failed for {GameId}", gameId);
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                // Expected when the game exits and the token is cancelled.
+            }
+        }
+
+        private static async Task StopHeartbeatAsync(CancellationTokenSource cancellationTokenSource, Task heartbeatTask)
+        {
+            cancellationTokenSource.Cancel();
+
+            if (heartbeatTask != null)
+            {
+                try
+                {
+                    await heartbeatTask;
+                }
+                catch (OperationCanceledException)
+                {
+                }
+            }
+        }
+
         public async Task Stop(Guid gameId)
         {
             if (_running.ContainsKey(gameId))

LANCommander.SDK/Rpc/Interfaces/Server/IRpcHub.GameKeepAlive.cs

@@ -0,0 +1,9 @@
+using System;
+using System.Threading.Tasks;
+
+namespace LANCommander.SDK.Rpc.Server;
+
+public partial interface IRpcHub
+{
+    Task GameKeepAliveAsync(Guid gameId);
+}

LANCommander.Server.Services/Extensions/IServiceCollectionExtensions.cs

@@ -98,6 +98,9 @@ public static class IServiceCollectionExtensions
 
         services.AddSingleton<ServerManager>();
 
+        services.AddSingleton<PlaySessionKeepAliveTracker>();
+        services.AddHostedService<PlaySessionSweepService>();
+
         services.AddSingleton<ScriptDebugger>();
         services.AddSingleton<IScriptDebugger>(sp =>
             sp.GetRequiredService<ScriptDebugger>());

LANCommander.Server.Services/PlaySessionKeepAliveTracker.cs

@@ -0,0 +1,50 @@
+using ZiggyCreatures.Caching.Fusion;
+
+namespace LANCommander.Server.Services;
+
+/// <summary>
+/// Tracks the last time each active play session was heard from via the RPC keepalive. Backed by
+/// <see cref="IFusionCache"/> so it works across instances when a distributed (e.g. Redis) backend
+/// is configured. The sweep consults this to end sessions that have gone stale, ending them at the
+/// last known-alive time so recorded playtime isn't inflated.
+/// </summary>
+public sealed class PlaySessionKeepAliveTracker(
+    IFusionCache cache,
+    SettingsProvider<Settings.Settings> settingsProvider)
+{
+    private static string GetCacheKey(Guid gameId, Guid userId) => $"PlaySessions/KeepAlive/{gameId}/{userId}";
+
+    // Keep entries alive well past the staleness timeout so the sweep can still read the last
+    // keepalive time when ending a stale session, rather than losing it to expiration.
+    private TimeSpan EntryDuration =>
+        TimeSpan.FromSeconds(Math.Max(1, settingsProvider.CurrentValue.Server.GameServers.KeepAliveTimeout) * 3);
+
+    /// <summary>Records that the session for this game/user was heard from just now.</summary>
+    public async Task TouchAsync(Guid gameId, Guid userId) =>
+        await cache.SetAsync(GetCacheKey(gameId, userId), DateTime.UtcNow, EntryDuration);
+
+    /// <summary>
+    /// Returns the last-seen time for the session, seeding it with the current time if we have no
+    /// record yet (e.g. a session that predates a restart with no distributed backend). Seeding
+    /// grants a fresh grace window rather than immediately reaping a session we haven't observed yet.
+    /// </summary>
+    public async Task<DateTime> GetOrSeedAsync(Guid gameId, Guid userId)
+    {
+        var cacheKey = GetCacheKey(gameId, userId);
+
+        var lastSeen = await cache.TryGetAsync<DateTime>(cacheKey);
+
+        if (lastSeen.HasValue)
+            return lastSeen.Value;
+
+        var now = DateTime.UtcNow;
+
+        await cache.SetAsync(cacheKey, now, EntryDuration);
+
+        return now;
+    }
+
+    /// <summary>Stops tracking the session (it ended normally or was swept).</summary>
+    public async Task RemoveAsync(Guid gameId, Guid userId) =>
+        await cache.RemoveAsync(GetCacheKey(gameId, userId));
+}

LANCommander.Server.Services/PlaySessionService.cs

@@ -16,6 +16,7 @@ namespace LANCommander.Server.Services
         IMapper mapper,
         IHttpContextAccessor httpContextAccessor,
         IDbContextFactory<DatabaseContext> contextFactory,
+        PlaySessionKeepAliveTracker keepAliveTracker,
         ServerService serverService) : BaseDatabaseService<PlaySession>(logger, settingsProvider, cache, mapper, httpContextAccessor, contextFactory)
     {
         public override async Task<PlaySession> AddAsync(PlaySession entity)
@@ -51,6 +52,8 @@ namespace LANCommander.Server.Services
             };
 
             await AddAsync(session);
+
+            await keepAliveTracker.TouchAsync(gameId, userId);
         }
 
         public async Task EndSessionAsync(Guid gameId, Guid userId)
@@ -63,6 +66,50 @@ namespace LANCommander.Server.Services
 
                 await UpdateAsync(existingSession);
             }
+
+            await keepAliveTracker.RemoveAsync(gameId, userId);
+        }
+
+        /// <summary>
+        /// Records that the player is still in the game. Tracked via the cache; the sweep uses it to
+        /// detect sessions that have gone silent.
+        /// </summary>
+        public async Task KeepAliveAsync(Guid gameId, Guid userId)
+        {
+            await keepAliveTracker.TouchAsync(gameId, userId);
+        }
+
+        /// <summary>
+        /// Ends any active session whose last keepalive is older than <paramref name="timeout"/>,
+        /// setting its End to that last known-alive time. Returns the affected games so callers can
+        /// schedule server autostop.
+        /// </summary>
+        public async Task<IEnumerable<Guid>> EndStaleSessionsAsync(TimeSpan timeout)
+        {
+            var cutoff = DateTime.UtcNow - timeout;
+
+            var activeSessions = await GetAsync(ps => ps.End == null);
+
+            var affectedGameIds = new HashSet<Guid>();
+
+            foreach (var session in activeSessions)
+            {
+                var lastSeen = await keepAliveTracker.GetOrSeedAsync(session.GameId.GetValueOrDefault(), session.UserId);
+
+                if (lastSeen >= cutoff)
+                    continue;
+
+                session.End = lastSeen;
+
+                await UpdateAsync(session);
+
+                await keepAliveTracker.RemoveAsync(session.GameId.GetValueOrDefault(), session.UserId);
+
+                if (session.GameId.HasValue)
+                    affectedGameIds.Add(session.GameId.Value);
+            }
+
+            return affectedGameIds;
         }
     }
 }

LANCommander.Server.Services/PlaySessionSweepService.cs

@@ -0,0 +1,49 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace LANCommander.Server.Services;
+
+/// <summary>
+/// Periodically ends play sessions that have stopped sending keepalives (e.g. the launcher crashed
+/// or lost its connection), so the active-session list stays accurate for autostop and player
+/// counts. Stale sessions are ended at their last known-alive time. Each affected game is handed to
+/// <see cref="ServerManager.ScheduleStop"/> so on-player-activity servers still autostop.
+/// </summary>
+public sealed class PlaySessionSweepService(
+    IServiceScopeFactory scopeFactory,
+    ServerManager serverManager,
+    SettingsProvider<Settings.Settings> settingsProvider,
+    ILogger<PlaySessionSweepService> logger) : BackgroundService
+{
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                var gameServers = settingsProvider.CurrentValue.Server.GameServers;
+                var interval = TimeSpan.FromSeconds(Math.Max(5, gameServers.KeepAliveSweepInterval));
+                var timeout = TimeSpan.FromSeconds(Math.Max(1, gameServers.KeepAliveTimeout));
+
+                await Task.Delay(interval, stoppingToken);
+
+                using var scope = scopeFactory.CreateScope();
+                var playSessionService = scope.ServiceProvider.GetRequiredService<PlaySessionService>();
+
+                var affectedGameIds = await playSessionService.EndStaleSessionsAsync(timeout);
+
+                foreach (var gameId in affectedGameIds)
+                    serverManager.ScheduleStop(gameId);
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Play session sweep failed");
+            }
+        }
+    }
+}

LANCommander.Server.Settings/Models/GameServerSettings.cs

@@ -10,6 +10,13 @@ public class GameServerSettings
     // Seconds to wait after the last player stops a game before stopping its
     // on-player-activity servers. Debounces stop/start thrash when players relaunch.
     public int AutostopDelay { get; set; } = 300;
+
+    // Seconds without a keepalive from the launcher before an active play session is considered
+    // stale and ended. Should comfortably exceed the launcher's keepalive interval.
+    public int KeepAliveTimeout { get; set; } = 120;
+
+    // Seconds between sweeps that end stale play sessions.
+    public int KeepAliveSweepInterval { get; set; } = 30;
     public IEnumerable<ServerEngineConfiguration> ServerEngines { get; set; } = 
     [
         new()

LANCommander.Server/Hubs/PlaySession.cs

@@ -0,0 +1,17 @@
+namespace LANCommander.Server.Hubs;
+
+public partial class RpcHub
+{
+    public async Task GameKeepAliveAsync(Guid gameId)
+    {
+        try
+        {
+            if (Guid.TryParse(Context.UserIdentifier, out var userId))
+                await keepAliveTracker.TouchAsync(gameId, userId);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to record keepalive for game {GameId}", gameId);
+        }
+    }
+}

LANCommander.Server/Hubs/_RpcHub.cs

@@ -16,7 +16,8 @@ public partial class RpcHub(
     ServerService serverService,
     GameService gameService,
     ScriptDebugger scriptDebugger,
-    ScriptClient scriptClient) : Hub<IRpcSubscriber>, IRpcHub
+    ScriptClient scriptClient,
+    PlaySessionKeepAliveTracker keepAliveTracker) : Hub<IRpcSubscriber>, IRpcHub
 {
     private string GetConnectionsCacheKey(string userIdentifier) => $"RPC/Connections/{userIdentifier}";