ace/Source/ACE.Server/Command/Handlers/AdminStatCommands.cs

using System;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;

using ACE.Common;
using ACE.Database;
using ACE.DatLoader;
using ACE.Entity;
using ACE.Entity.Enum;
using ACE.Server.Managers;
using ACE.Server.Network;
using ACE.Server.Network.Managers;
using ACE.Server.Physics.Entity;
using ACE.Server.Physics.Managers;
using ACE.Server.WorldObjects;

using log4net;

namespace ACE.Server.Command.Handlers
{
    public static class AdminStatCommands
    {
        private static readonly ILog log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);

        // allstats
        [CommandHandler("allstats", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of all server statistics and usage")]
        public static void HandleAllStats(Session session, params string[] parameters)
        {
            HandleServerStatus(session, parameters);

            HandleServerPerformance(session, parameters);

            HandleLandblockStats(session, parameters);

            HandleLBGroupStats(session, parameters);

            HandleGCStatus(session, parameters);

            DeveloperDatabaseCommands.HandleDatabaseQueueInfo(session, parameters);
        }

        // serverstatus
        [CommandHandler("serverstatus", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of server statistics and usage")]
        public static void HandleServerStatus(Session session, params string[] parameters)
        {
            // This is formatted very similarly to GDL.

            var sb = new StringBuilder();

            var proc = Process.GetCurrentProcess();

            sb.Append($"Server Status:{'\n'}");

            sb.Append($"Host Info: {Environment.OSVersion}, vCPU: {Environment.ProcessorCount}{'\n'}");

            ThreadPool.GetMinThreads(out var minWorkerThreads, out var minCompletionPortThreads);
            ThreadPool.GetMaxThreads(out var maxWorkerThreads, out var maxCompletionPortThreads);
            ThreadPool.GetAvailableThreads(out var availWorkerThreads, out var availCompletionPortThreads);

            sb.Append($"ThreadPool Min: {minWorkerThreads} {minCompletionPortThreads}, Max: {maxWorkerThreads} {maxCompletionPortThreads}, Avail: {availWorkerThreads} {availCompletionPortThreads}, Current: {ThreadPool.ThreadCount}{'\n'}");

            var runTime = DateTime.Now - proc.StartTime;
            sb.Append($"Server Runtime: {(int)runTime.TotalHours}h {runTime.Minutes}m {runTime.Seconds}s{'\n'}");

            sb.Append($"Total CPU Time: {(int)proc.TotalProcessorTime.TotalHours}h {proc.TotalProcessorTime.Minutes}m {proc.TotalProcessorTime.Seconds}s, Threads: {proc.Threads.Count}{'\n'}");

            // todo, add actual system memory used/avail
            sb.Append($"{(proc.PrivateMemorySize64 >> 20):N0} MB used{'\n'}");  // sb.Append($"{(proc.PrivateMemorySize64 >> 20)} MB used, xxxx / yyyy MB physical mem free.{'\n'}");

            sb.Append($"{NetworkManager.GetSessionCount():N0} connections, {NetworkManager.GetAuthenticatedSessionCount():N0} authenticated connections, {NetworkManager.GetUniqueSessionEndpointCount():N0} unique connections, {PlayerManager.GetOnlineCount():N0} players online{'\n'}");
            sb.Append($"Total Accounts Created: {DatabaseManager.Authentication.GetAccountCount():N0}, Total Characters Created: {(PlayerManager.GetOfflineCount() + PlayerManager.GetOnlineCount()):N0}{'\n'}");

            // 330 active objects, 1931 total objects(16777216 buckets.)

            // todo, expand this
            var loadedLandblocks = LandblockManager.GetLoadedLandblocks();
            int dormantLandblocks = 0, activeDungeonLandblocks = 0, dormantDungeonLandblocks = 0;
            int players = 0, creatures = 0, missiles = 0, other = 0, total = 0;
            foreach (var landblock in loadedLandblocks)
            {
                if (landblock.IsDormant)
                    dormantLandblocks++;

                if (landblock.IsDungeon)
                {
                    if (landblock.IsDormant)
                        dormantDungeonLandblocks++;
                    else
                        activeDungeonLandblocks++;
                }

                foreach (var worldObject in landblock.GetAllWorldObjectsForDiagnostics())
                {
                    if (worldObject is Player)
                        players++;
                    else if (worldObject is Creature)
                        creatures++;
                    else if (worldObject.Missile ?? false)
                        missiles++;
                    else
                        other++;

                    total++;
                }
            }
            sb.Append($"Landblocks: {(loadedLandblocks.Count - dormantLandblocks):N0} active ({activeDungeonLandblocks:N0} dungeons), {dormantLandblocks:N0} dormant ({dormantDungeonLandblocks:N0} dungeons), Landblock Groups: {LandblockManager.LandblockGroupsCount:N0} - Players: {players:N0}, Creatures: {creatures:N0}, Missiles: {missiles:N0}, Other: {other:N0}, Total: {total:N0}.{'\n'}"); // 11 total blocks loaded. 11 active. 0 pending dormancy. 0 dormant. 314 unloaded.
            // 11 total blocks loaded. 11 active. 0 pending dormancy. 0 dormant. 314 unloaded.

            if (ServerPerformanceMonitor.IsRunning)
                sb.Append($"Server Performance Monitor - UpdateGameWorld ~5m {ServerPerformanceMonitor.GetEventHistory5m(ServerPerformanceMonitor.MonitorType.UpdateGameWorld_Entire).AverageEventDuration:N3}, ~1h {ServerPerformanceMonitor.GetEventHistory1h(ServerPerformanceMonitor.MonitorType.UpdateGameWorld_Entire).AverageEventDuration:N3} s{'\n'}");
            else
                sb.Append($"Server Performance Monitor - Not running. To start use /serverperformance start{'\n'}");

            sb.Append($"Threading - WorldThreadCount: {ConfigManager.Config.Server.Threading.LandblockManagerParallelOptions.MaxDegreeOfParallelism}, Multithread Physics: {ConfigManager.Config.Server.Threading.MultiThreadedLandblockGroupPhysicsTicking}, Multithread Non-Physics: {ConfigManager.Config.Server.Threading.MultiThreadedLandblockGroupTicking}, DatabaseThreadCount: {ConfigManager.Config.Server.Threading.DatabaseParallelOptions.MaxDegreeOfParallelism}{'\n'}");

            sb.Append($"Physics Cache Counts - BSPCache: {BSPCache.Count:N0}, GfxObjCache: {GfxObjCache.Count:N0}, PolygonCache: {PolygonCache.Count:N0}, VertexCache: {VertexCache.Count:N0}{'\n'}");

            sb.Append($"Total Server Objects: {ServerObjectManager.ServerObjects.Count:N0}{'\n'}");

            sb.Append($"World DB Cache Counts - Weenies: {DatabaseManager.World.GetWeenieCacheCount():N0}, LandblockInstances: {DatabaseManager.World.GetLandblockInstancesCacheCount():N0}, PointsOfInterest: {DatabaseManager.World.GetPointsOfInterestCacheCount():N0}, Cookbooks: {DatabaseManager.World.GetCookbookCacheCount():N0}, Spells: {DatabaseManager.World.GetSpellCacheCount():N0}, Encounters: {DatabaseManager.World.GetEncounterCacheCount():N0}, Events: {DatabaseManager.World.GetEventsCacheCount():N0}{'\n'}");
            //sb.Append($"Shard DB Counts - Biotas: {DatabaseManager.Shard.BaseDatabase.GetBiotaCount():N0}{'\n'}");
            sb.Append($"Shard DB Counts - Biotas: ~{DatabaseManager.Shard.BaseDatabase.GetEstimatedBiotaCount(ConfigManager.Config.MySql.Shard.Database):N0}{'\n'}");
            if (DatabaseManager.Shard.BaseDatabase is ShardDatabaseWithCaching shardDatabaseWithCaching)
            {
                var biotaIds = shardDatabaseWithCaching.GetBiotaCacheKeys();
                var playerBiotaIds = biotaIds.Count(id => ObjectGuid.IsPlayer(id));
                var nonPlayerBiotaIds = biotaIds.Count - playerBiotaIds;
                sb.Append($"Shard DB Cache Counts - Player Biotas: {playerBiotaIds} ~ {shardDatabaseWithCaching.PlayerBiotaRetentionTime.TotalMinutes:N0} m, Non Players {nonPlayerBiotaIds} ~ {shardDatabaseWithCaching.NonPlayerBiotaRetentionTime.TotalMinutes:N0} m{'\n'}");
            }

            sb.Append(GuidManager.GetDynamicGuidDebugInfo() + '\n');

            sb.Append($"Portal.dat has {DatManager.PortalDat.FileCache.Count:N0} files cached of {DatManager.PortalDat.AllFiles.Count:N0} total{'\n'}");
            sb.Append($"Cell.dat has {DatManager.CellDat.FileCache.Count:N0} files cached of {DatManager.CellDat.AllFiles.Count:N0} total{'\n'}");

            CommandHandlerHelper.WriteOutputInfo(session, $"{sb}");
        }

        // serverperformance
        [CommandHandler("serverperformance", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of server performance statistics")]
        public static void HandleServerPerformance(Session session, params string[] parameters)
        {
            if (parameters != null && (parameters.Length == 1 || parameters.Length == 2))
            {
                if (parameters.Length >= 1 && parameters[0].ToLower() == "start")
                {
                    if (parameters.Length >= 2 && parameters[1].ToLower() == "cumulative")
                    {
                        ServerPerformanceMonitor.StartCumulative();
                        CommandHandlerHelper.WriteOutputInfo(session, "Cumulative Server Performance Monitor started");
                        return;
                    }
                    else
                    {
                        ServerPerformanceMonitor.Start();
                        CommandHandlerHelper.WriteOutputInfo(session, "Server Performance Monitor started");
                        return;
                    }
                }

                if (parameters.Length >= 1 && parameters[0].ToLower() == "stop")
                {
                    if (parameters.Length >= 2 && parameters[1].ToLower() == "cumulative")
                    {
                        ServerPerformanceMonitor.StopCumulative();
                        CommandHandlerHelper.WriteOutputInfo(session, "Cumulative Server Performance Monitor stopped");
                        return;
                    }
                    else
                    {
                        ServerPerformanceMonitor.Stop();
                        CommandHandlerHelper.WriteOutputInfo(session, "Server Performance Monitor stopped");
                        return;
                    }
                }

                if (parameters.Length >= 1 && parameters[0].ToLower() == "reset")
                {
                    ServerPerformanceMonitor.Reset();
                    CommandHandlerHelper.WriteOutputInfo(session, "Server Performance Monitor reset");
                    return;
                }
            }

            if (!ServerPerformanceMonitor.IsRunning)
            {
                CommandHandlerHelper.WriteOutputInfo(session, "Server Performance Monitor not running. To start use /serverperformance start");
                return;
            }

            CommandHandlerHelper.WriteOutputInfo(session, ServerPerformanceMonitor.ToString());
        }

        [CommandHandler("landblockperformance", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of landblock performance statistics")]
        [CommandHandler("landblockstats", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of landblock performance statistics")]
        public static void HandleLandblockStats(Session session, params string[] parameters)
        {
            var sb = new StringBuilder();

            var loadedLandblocks = LandblockManager.GetLoadedLandblocks();

            // Filter out landblocks that haven't recorded a certain amount of events
            var sortedBy5mAverage = loadedLandblocks.Where(r => r.Monitor5m.EventHistory.TotalEvents >= 10).OrderByDescending(r => r.Monitor5m.EventHistory.AverageEventDuration).Take(10).ToList();
            var sortedBy1hrAverage = loadedLandblocks.Where(r => r.Monitor1h.EventHistory.TotalEvents >= 1000).OrderByDescending(r => r.Monitor1h.EventHistory.AverageEventDuration).Take(10).ToList();

            var combinedByAverage = sortedBy5mAverage.Concat(sortedBy1hrAverage).Distinct().OrderByDescending(r => Math.Max(r.Monitor5m.EventHistory.AverageEventDuration, r.Monitor1h.EventHistory.AverageEventDuration)).Take(10);

            sb.Append($"Most Busy Landblock - By Average{'\n'}");
            sb.Append($"~5m Hits   Avg  Long  Last - ~1h Hits   Avg  Long  Last - Location   Players  Creatures{'\n'}");

            foreach (var entry in combinedByAverage)
            {
                int players = 0, creatures = 0;
                foreach (var worldObject in entry.GetAllWorldObjectsForDiagnostics())
                {
                    if (worldObject is Player)
                        players++;
                    else if (worldObject is Creature)
                        creatures++;
                }

                sb.Append($"{entry.Monitor5m.EventHistory.TotalEvents.ToString().PadLeft(7)} {entry.Monitor5m.EventHistory.AverageEventDuration:N4} {entry.Monitor5m.EventHistory.LongestEvent:N3} {entry.Monitor5m.EventHistory.LastEvent:N3} - " +
                          $"{entry.Monitor1h.EventHistory.TotalEvents.ToString().PadLeft(7)} {entry.Monitor1h.EventHistory.AverageEventDuration:N4} {entry.Monitor1h.EventHistory.LongestEvent:N3} {entry.Monitor1h.EventHistory.LastEvent:N3} - " +
                          $"0x{entry.Id.Raw:X8} {players.ToString().PadLeft(7)}  {creatures.ToString().PadLeft(9)}{'\n'}");
            }

            var sortedBy5mLong = loadedLandblocks.OrderByDescending(r => r.Monitor5m.EventHistory.LongestEvent).Take(10);
            var sortedBy1hrLong = loadedLandblocks.OrderByDescending(r => r.Monitor1h.EventHistory.LongestEvent).Take(10);

            var combinedByLong = sortedBy5mLong.Concat(sortedBy1hrLong).Distinct().OrderByDescending(r => Math.Max(r.Monitor5m.EventHistory.LongestEvent, r.Monitor1h.EventHistory.LongestEvent)).Take(10);

            sb.Append($"Most Busy Landblock - By Longest{'\n'}");
            sb.Append($"~5m Hits   Avg  Long  Last - ~1h Hits   Avg  Long  Last - Location   Players  Creatures{'\n'}");

            foreach (var entry in combinedByLong)
            {
                int players = 0, creatures = 0;
                foreach (var worldObject in entry.GetAllWorldObjectsForDiagnostics())
                {
                    if (worldObject is Player)
                        players++;
                    else if (worldObject is Creature)
                        creatures++;
                }

                sb.Append($"{entry.Monitor5m.EventHistory.TotalEvents.ToString().PadLeft(7)} {entry.Monitor5m.EventHistory.AverageEventDuration:N4} {entry.Monitor5m.EventHistory.LongestEvent:N3} {entry.Monitor5m.EventHistory.LastEvent:N3} - " +
                          $"{entry.Monitor1h.EventHistory.TotalEvents.ToString().PadLeft(7)} {entry.Monitor1h.EventHistory.AverageEventDuration:N4} {entry.Monitor1h.EventHistory.LongestEvent:N3} {entry.Monitor1h.EventHistory.LastEvent:N3} - " +
                          $"0x{entry.Id.Raw:X8} {players.ToString().PadLeft(7)}  {creatures.ToString().PadLeft(9)}{'\n'}");
            }

            CommandHandlerHelper.WriteOutputInfo(session, sb.ToString());
        }

        // lbgroupstats
        [CommandHandler("lbgroupstats", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of landblock group stats")]
        public static void HandleLBGroupStats(Session session, params string[] parameters)
        {
            var sb = new StringBuilder();

            sb.Append($"TickPhysicsEfficiencyTracker: {LandblockManager.TickPhysicsEfficiencyTracker.AverageAmount,3:N0} %, TickMultiThreadedWorkEfficiencyTracker: {LandblockManager.TickMultiThreadedWorkEfficiencyTracker.AverageAmount,3:N0} %{'\n'}");

            var loadedLanblockGroups = LandblockManager.GetLoadedLandblockGroups();

            var sortedByLargest = loadedLanblockGroups.OrderByDescending(r => r.Count).Take(5);

            sb.Append($"Largest Landblock Groups{'\n'}");
            sb.Append($"Cnt, XMin - XMax, YMin - YMax, TickPhysicsTracker avg   max, TickMultiThreadedWorkTracker avg   max (s){'\n'}");

            foreach (var landblockGroup in sortedByLargest)
                sb.Append($"{landblockGroup.Count,3},   {landblockGroup.XMin,2:X2} - {landblockGroup.XMax,2:X2},     {landblockGroup.YMin,2:X2} - {landblockGroup.YMax,2:X2}  ,                  {landblockGroup.TickPhysicsTracker.AverageAmount,5:N3} {landblockGroup.TickPhysicsTracker.LargestAmount,5:N3},                            {landblockGroup.TickMultiThreadedWorkTracker.AverageAmount,5:N3} {landblockGroup.TickMultiThreadedWorkTracker.LargestAmount,5:N3}{'\n'}");

            var sortedByTopTickPhysicsTracker = loadedLanblockGroups.OrderByDescending(r => r.TickPhysicsTracker.AverageAmount).Take(5);

            sb.Append($"Top TickPhysicsTracker Landblock Groups{'\n'}");

            foreach (var landblockGroup in sortedByTopTickPhysicsTracker)
                sb.Append($"{landblockGroup.Count,3},   {landblockGroup.XMin,2:X2} - {landblockGroup.XMax,2:X2},     {landblockGroup.YMin,2:X2} - {landblockGroup.YMax,2:X2}  ,                  {landblockGroup.TickPhysicsTracker.AverageAmount,5:N3} {landblockGroup.TickPhysicsTracker.LargestAmount,5:N3},                            {landblockGroup.TickMultiThreadedWorkTracker.AverageAmount,5:N3} {landblockGroup.TickMultiThreadedWorkTracker.LargestAmount,5:N3}{'\n'}");

            var sortedByTopTickMultiThreadedWorkTracker = loadedLanblockGroups.OrderByDescending(r => r.TickMultiThreadedWorkTracker.AverageAmount).Take(5);

            sb.Append($"Top TickMultiThreadedWorkTracker Landblock Groups{'\n'}");

            foreach (var landblockGroup in sortedByTopTickMultiThreadedWorkTracker)
                sb.Append($"{landblockGroup.Count,3},   {landblockGroup.XMin,2:X2} - {landblockGroup.XMax,2:X2},     {landblockGroup.YMin,2:X2} - {landblockGroup.YMax,2:X2}  ,                  {landblockGroup.TickPhysicsTracker.AverageAmount,5:N3} {landblockGroup.TickPhysicsTracker.LargestAmount,5:N3},                            {landblockGroup.TickMultiThreadedWorkTracker.AverageAmount,5:N3} {landblockGroup.TickMultiThreadedWorkTracker.LargestAmount,5:N3}{'\n'}");

            CommandHandlerHelper.WriteOutputInfo(session, sb.ToString());
        }

        // gcstatus
        [CommandHandler("gcstatus", AccessLevel.Advocate, CommandHandlerFlag.None, 0, "Displays a summary of server GC Information")]
        public static void HandleGCStatus(Session session, params string[] parameters)
        {
            var sb = new StringBuilder();

            sb.Append($"GC.GetTotalMemory: {(GC.GetTotalMemory(false) >> 20):N0} MB, GC.GetTotalAllocatedBytes: {(GC.GetTotalAllocatedBytes() >> 20):N0} MB{'\n'}");

            // https://docs.microsoft.com/en-us/dotnet/api/system.gcmemoryinfo?view=net-5.0
            var gcmi = GC.GetGCMemoryInfo();

            sb.Append($"GCMI Index: {gcmi.Index:N0}, Generation: {gcmi.Generation}, Compacted: {gcmi.Compacted}, Concurrent: {gcmi.Concurrent}, PauseTimePercentage: {gcmi.PauseTimePercentage}{'\n'}");
            for (int i = 0 ; i < gcmi.GenerationInfo.Length ; i++)
                sb.Append($"GCMI.GenerationInfo[{i}] FragmentationBeforeBytes: {(gcmi.GenerationInfo[i].FragmentationBeforeBytes >> 20):N0} MB, FragmentationAfterBytes: {(gcmi.GenerationInfo[i].FragmentationAfterBytes >> 20):N0} MB, SizeBeforeBytes: {(gcmi.GenerationInfo[i].SizeBeforeBytes >> 20):N0} MB, SizeAfterBytes: {(gcmi.GenerationInfo[i].SizeAfterBytes >> 20):N0} MB{'\n'}");
            for (int i = 0; i < gcmi.PauseDurations.Length; i++)
                sb.Append($"GCMI.PauseDurations[{i}]: {gcmi.PauseDurations[i].TotalMilliseconds:N0} ms{'\n'}");
            sb.Append($"GCMI PinnedObjectsCount: {gcmi.PinnedObjectsCount}, FinalizationPendingCount: {gcmi.FinalizationPendingCount:N0}{'\n'}");

            sb.Append($"GCMI FragmentedBytes: {(gcmi.FragmentedBytes >> 20):N0} MB, PromotedBytes: {(gcmi.PromotedBytes >> 20):N0} MB, HeapSizeBytes: {(gcmi.HeapSizeBytes >> 20):N0} MB, TotalCommittedBytes: {(gcmi.TotalCommittedBytes >> 20):N0} MB{'\n'}");
            sb.Append($"GCMI MemoryLoadBytes: {(gcmi.MemoryLoadBytes >> 20):N0} MB, HighMemoryLoadThresholdBytes: {(gcmi.HighMemoryLoadThresholdBytes >> 20):N0} MB, TotalAvailableMemoryBytes: {(gcmi.TotalAvailableMemoryBytes >> 20):N0} MB{'\n'}");

            CommandHandlerHelper.WriteOutputInfo(session, sb.ToString());
        }
    }
}