From 3f89ddd11fddefcbd6592ec99fac8543ab35510c Mon Sep 17 00:00:00 2001 From: wdunn001 Date: Mon, 3 Aug 2026 12:55:53 -0400 Subject: [PATCH] community: fix Spire login path, add staleness cutoff for DAoC serverstats - Spire's auth controller is mounted at bare /auth, not /api/v1/login or /api/v1/auth/login (confirmed live via Spire's own unauthenticated GET /api/v1/routes introspection endpoint). Two earlier guesses both 405'd. Default login_path corrected to /auth/login. - mysql_direct gains value_column/observed_at_column/max_staleness_sec: OpenDAoC's serverstats is a periodic snapshot (written once a minute by the server's own StatSave.cs), not a live aggregate, so a query must read the newest row and discard it as unavailable once it's too old rather than presenting a stale reading -- possibly from a since-crashed server -- as a current live count. - Registry updated for both now-confirmed-live pieces: daoc-mysql's serverproperty.statsave_interval is enabled (1 minute) and verified writing; the drop-service Spire account exists (user ID 3), credentials in OpenBao kv/eqemu/spire, materialized host-side into /opt/drop/spire-service.env (not yet mounted -- compose change is separate, see homelab-compose). --- .../community/data/gameServerRegistry.json | 15 +++-- .../internal/community/serverProbeService.ts | 58 ++++++++++++++++++- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/server/server/internal/community/data/gameServerRegistry.json b/server/server/internal/community/data/gameServerRegistry.json index b3b8e415..ad65643d 100644 --- a/server/server/internal/community/data/gameServerRegistry.json +++ b/server/server/internal/community/data/gameServerRegistry.json @@ -31,17 +31,17 @@ "join_instructions": "Point an EQEmu-compatible RoF2-era client (with eqemu-loader or equivalent) at host everquest.quasarke.net. Login server: 5998/tcp (Titanium protocol) or 5999/udp (SoD+/RoF2 protocol). Zone ports (7000-7400, plus 7778/udp) are not published individually and must not be probed directly -- they're DNAT'd through automatically once a character enters the world.", "kind": "eqemu", "name": "Quasarke EverQuest (RoF2-era rules, EQEmu)", - "notes": "Spire (admin/status panel) runs supervising world/zone/loginserver INSIDE the same eqemu-server container (one container by design, see /opt/eqemu-docker/docker-compose.yml) and is reverse-proxied at https://everquest.quasarke.net from 127.0.0.1:8211 on .88. IaC: homelab-compose eqemu-stack / eqemu-docker-stack / eqemu-edge-stack / spire-docker (all present, no drift found). DEEP LINK resolved 2026-08-03: \"EverQuest\" -> Game.id abc35ffd-8af5-4cf6-958e-b2d9ebca3892. PLAYER COUNT CORRECTED 2026-08-03 -- an earlier claim that this was 'genuinely unavailable' was WRONG about there being no source; it is right that there is no DB-backed source (re-verified live against the peq schema: no who_all_cache table, no characters.online-equivalent column on character_data, account/login_world_servers/spire_analytic_events carry no online-player count -- checked all four before concluding). The REAL source, found by reading Spire's own source (github mirror mmo-ref/eqemu-spire): World's telnet admin console (127.0.0.1:9001 inside eqemu-server, loopback-only, unauthenticated-by-IP by EQEmu's own design -- must NOT be exposed on mmo-net) answers `api get_server_counts` with the live in-memory client count, and Spire wraps that safely behind a real login at `GET /api/v1/eqemuserver/server-stats` -> `{\"players_online\": N}`. That endpoint 401s without a session (verified live, 'You are not logged in') -- Spire's BASIC_AUTH_USER/PASSWORD env var is a separate outer HTTP gate and does NOT satisfy the app's own session check, so it is not a shortcut here. The probe therefore logs in as a dedicated Spire service account (`spire user:create`, not yet provisioned as of this sweep -- see homelab-compose eqemu-docker-stack notes) and reuses the JWT, the same as a human operator's browser session. Until that account exists, credentials_file is absent and this correctly reports unavailable, never a fabricated count.", + "notes": "Spire (admin/status panel) runs supervising world/zone/loginserver INSIDE the same eqemu-server container (one container by design, see /opt/eqemu-docker/docker-compose.yml) and is reverse-proxied at https://everquest.quasarke.net from 127.0.0.1:8211 on .88. IaC: homelab-compose eqemu-stack / eqemu-docker-stack / eqemu-edge-stack / spire-docker (all present, no drift found). DEEP LINK resolved 2026-08-03: \"EverQuest\" -> Game.id abc35ffd-8af5-4cf6-958e-b2d9ebca3892. PLAYER COUNT CORRECTED 2026-08-03 -- an earlier claim that this was 'genuinely unavailable' was WRONG about there being no source; it is right that there is no DB-backed source (re-verified live against the peq schema: no who_all_cache table, no characters.online-equivalent column on character_data, account/login_world_servers/spire_analytic_events carry no online-player count -- checked all four before concluding). The REAL source, found by reading Spire's own source (github mirror mmo-ref/eqemu-spire): World's telnet admin console (127.0.0.1:9001 inside eqemu-server, loopback-only, unauthenticated-by-IP by EQEmu's own design -- must NOT be exposed on mmo-net) answers `api get_server_counts` with the live in-memory client count, and Spire wraps that safely behind a real login at `GET /api/v1/eqemuserver/server-stats` -> `{\"players_online\": N}`. That endpoint 401s without a session (verified live, 'You are not logged in') -- Spire's BASIC_AUTH_USER/PASSWORD env var is a separate outer HTTP gate and does NOT satisfy the app's own session check, so it is not a shortcut here. PROVISIONED 2026-08-03: a dedicated low-privilege Spire local account `drop-service` was created via `docker exec eqemu-server spire user:create drop-service ` (user ID 3); the generated password is stored in OpenBao kv/eqemu/spire (SPIRE_SERVICE_USER, SPIRE_SERVICE_PASSWORD, SPIRE_BASE_URL), never in git, never seen by the agent that requested it. The login route is `POST /auth/login` (NOT under /api/v1 -- confirmed 2026-08-03 via Spire's own unauthenticated `GET /api/v1/routes` introspection; two earlier path guesses, /api/v1/user/login and /api/v1/auth/login, both 405'd). World's telnet console itself (127.0.0.1:9001, unauthenticated-by-IP) is deliberately never contacted directly or exposed on mmo-net -- only Spire's authenticated HTTP wrapper around it is used. credentials_file must be materialized host-side from the OpenBao path above into /opt/drop/spire-service.env before this probe can report a real count; until that file exists it correctly reports unavailable, never a fabricated count.", "player_count_probe": { "method": "spire_session_api", "base_url": "https://everquest.quasarke.net", - "login_path": "/api/v1/login", + "login_path": "/auth/login", "stats_path": "/api/v1/eqemuserver/server-stats", "json_field": "players_online", "credentials_file": "/run/secrets/spire-service.env", "user_env_var": "SPIRE_SERVICE_USER", "password_env_var": "SPIRE_SERVICE_PASSWORD", - "note": "Requires a dedicated low-privilege Spire local account created via `docker exec eqemu-server spire user:create ` on .88, with the resulting credentials placed in the mounted /run/secrets/spire-service.env (never committed, never routed through an LLM session -- see homelab-compose eqemu-docker-stack for provisioning notes). Not yet provisioned as of 2026-08-03; until then credentials_file is missing and probePlayerCount reports 'not fully configured', which surfaces as unavailable, not a fabricated count. World's telnet console itself (127.0.0.1:9001, unauthenticated-by-IP) is deliberately never contacted directly or exposed on mmo-net -- only Spire's authenticated HTTP wrapper around it is used." + "note": "credentials_file is a read-only bind mount of /opt/drop/spire-service.env on .88, materialized host-side from OpenBao kv/eqemu/spire (SPIRE_SERVICE_USER/SPIRE_SERVICE_PASSWORD) -- values never routed through an LLM session. Spire account `drop-service` (user ID 3) was created via `docker exec eqemu-server spire user:create` on 2026-08-03; see homelab-compose/docs/game-server-population-probes.md for the durable record. World's telnet console itself (127.0.0.1:9001, unauthenticated-by-IP) is deliberately never contacted directly or exposed on mmo-net -- only Spire's authenticated HTTP wrapper around it is used." }, "port": 5998, "probe_config": { @@ -99,7 +99,7 @@ "join_instructions": "Point the DOLLoader at 192.168.1.88 port 10310 (not 10300): `DOLLoader.exe 192.168.1.88 10310 1 `. Account self-service is not yet confirmed -- the compose file sets AUTO_ACCOUNT_CREATION=False on purpose (accounts are meant to come from mmo.quasarke.net, the same portal fronting EQEmu/WoW), so verify the actual DAoC signup path on mmo.quasarke.net before publishing this entry to players.", "kind": "generic", "name": "Quasarke Dark Age of Camelot (OpenDAoC)", - "notes": "FOUND RUNNING during this sweep (2026-08-01) -- not previously known to any registry effort, and not mentioned in the task's server list. Containers daoc-mysql (mariadb:11.4, healthy, loopback-only 127.0.0.1:3310) and daoc-server (opendaoc:local, host-networked) are up on .88. IaC GAP: homelab-compose/daoc-stack/ (docker-compose.yml + Dockerfile + scripts/) exists in the local homelab-compose worktree but is UNTRACKED in git -- not committed to any branch, not backed by durable IaC yet. No daoc.quasarke.net Caddy vhost exists -- OpenDAoC needs a raw TCP/UDP client connection so it can't ride the reverse proxy the way wow/everquest do. deep_link left null: no Drop catalog entry exists for this client yet. PLAYER COUNT REPLACED 2026-08-03: the old ssh_docker_logs_regex hack (tailing daoc-server's stdout for a StatPrint 'Clients: N' line over ssh) is retired. There is no deployed OpenDAoC management UI (checked -- none of AccountManager/QueueService/RedBot-Cogs run here), but OpenDAoC-Core ITSELF ships a first-party mechanism, found by reading its own upstream source (github.com/OpenDAoC/OpenDAoC-Core): GameServer/gameutils/StatSave.cs writes ClientService.Instance.ClientCount (plus per-realm AlbionPlayers/MidgardPlayers/HiberniaPlayers) into a `serverstats` DB table every server.statsave_interval minutes, and upstream's own companion GameServer/scripts/system/stats.php reads it back with the same shape of query used below. Verified live against daoc-mysql's schema: the `serverstats` table exists with exactly those columns, but is currently EMPTY because `serverproperty.statsave_interval = 0` (disabled) on this deploy -- a supported feature simply switched off, not a missing capability. Flipping it to a positive value (minutes) is a pending, confirmed-needed follow-up (single-row UPDATE + daoc-server restart to pick it up); until then this probe correctly returns 'no rows' -> unavailable, never a fabricated 0.", + "notes": "FOUND RUNNING during this sweep (2026-08-01) -- not previously known to any registry effort, and not mentioned in the task's server list. Containers daoc-mysql (mariadb:11.4, healthy, loopback-only 127.0.0.1:3310) and daoc-server (opendaoc:local, host-networked) are up on .88. IaC GAP: homelab-compose/daoc-stack/ (docker-compose.yml + Dockerfile + scripts/) exists in the local homelab-compose worktree but is UNTRACKED in git -- not committed to any branch, not backed by durable IaC yet. No daoc.quasarke.net Caddy vhost exists -- OpenDAoC needs a raw TCP/UDP client connection so it can't ride the reverse proxy the way wow/everquest do. deep_link left null: no Drop catalog entry exists for this client yet. PLAYER COUNT REPLACED 2026-08-03: the old ssh_docker_logs_regex hack (tailing daoc-server's stdout for a StatPrint 'Clients: N' line over ssh) is retired. There is no deployed OpenDAoC management UI (checked -- none of AccountManager/QueueService/RedBot-Cogs run here), but OpenDAoC-Core ITSELF ships a first-party mechanism, found by reading its own upstream source (github.com/OpenDAoC/OpenDAoC-Core): GameServer/gameutils/StatSave.cs writes ClientService.Instance.ClientCount (plus per-realm AlbionPlayers/MidgardPlayers/HiberniaPlayers) into a `serverstats` DB table every server.statsave_interval minutes, and upstream's own companion GameServer/scripts/system/stats.php reads it back with the same shape of query used below. `serverproperty.statsave_interval` was 0 (disabled) on this deploy -- ENABLED 2026-08-03 (set to 1 minute) and daoc-server restarted (0 players online at the time, no interruption); serverstats confirmed writing a fresh row every minute, e.g. Clients=0/AlbionPlayers=0/MidgardPlayers=0/HiberniaPlayers=0 at 16:50:58 -- a trustworthy real zero, not a placeholder. Durable record of the statsave_interval=1 setting: homelab-compose/docs/game-server-population-probes.md (daoc-stack itself is still untracked, pending a merge from other concurrent work, so this setting is NOT recorded inside it -- if daoc-stack is ever rebuilt from a fresh dump, statsave_interval must be re-applied per that doc). Because serverstats is a periodic snapshot rather than a live aggregate, the probe reads the newest row and discards it as unavailable if it's older than max_staleness_sec, rather than presenting a stale reading (possibly from a since-crashed server) as a current live 0.", "player_count_probe": { "method": "mysql_direct", "host": "daoc-mysql", @@ -108,8 +108,11 @@ "credentials_file": "/run/secrets/daoc-db.env", "user": "opendaoc", "password_env_var": "DAOC_DB_PASSWORD", - "query": "SELECT Clients FROM serverstats ORDER BY StatDate DESC LIMIT 1", - "note": "credentials_file is a read-only bind mount of the same /opt/daoc/.env the daoc-mysql container already uses (DAOC_DB_PASSWORD); the DB username is the literal 'opendaoc' set directly in homelab-compose/daoc-stack/docker-compose.yml (MARIADB_USER: opendaoc), not a secret. host is the daoc-mysql container name, reachable now that drop shares mmo-net with it. Query and source table match OpenDAoC-Core's own StatSave.cs/stats.php exactly. Requires serverproperty.statsave_interval > 0 on daoc-mysql (currently 0/disabled) -- see notes; until enabled this returns no rows and the probe honestly reports unavailable." + "query": "SELECT Clients, StatDate FROM serverstats ORDER BY StatDate DESC LIMIT 1", + "value_column": "Clients", + "observed_at_column": "StatDate", + "max_staleness_sec": 150, + "note": "credentials_file is a read-only bind mount of the same /opt/daoc/.env the daoc-mysql container already uses (DAOC_DB_PASSWORD); the DB username is the literal 'opendaoc' set directly in homelab-compose/daoc-stack/docker-compose.yml (MARIADB_USER: opendaoc), not a secret. host is the daoc-mysql container name, reachable now that drop shares mmo-net with it. Query and source table match OpenDAoC-Core's own StatSave.cs/stats.php exactly. serverproperty.statsave_interval=1 (minute) enabled 2026-08-03, confirmed writing live. max_staleness_sec=150 is 2.5x the 60s save interval -- generous enough to absorb one missed tick, tight enough that a genuinely dead server (whose last serverstats row stops advancing) reads unavailable within ~2.5 minutes instead of showing its last-known count forever." }, "port": 10310, "probe_config": { diff --git a/server/server/internal/community/serverProbeService.ts b/server/server/internal/community/serverProbeService.ts index 8b1f292c..7443a636 100644 --- a/server/server/internal/community/serverProbeService.ts +++ b/server/server/internal/community/serverProbeService.ts @@ -38,11 +38,17 @@ import type { GameServer, Prisma } from "~/prisma/client/client"; // - OpenDAoC: upstream OpenDAoC-Core's own StatSave.cs writes ClientCount // into `serverstats.Clients` every server.statsave_interval minutes; its // own companion stats.php reads it back the same way this does. +// statsave_interval was 0 (disabled) on this deploy until 2026-08-03; it +// is now 1 (minute), so a query must read the LATEST row and treat one +// older than a couple of intervals as unavailable, not as a live 0 -- +// see mysql_direct's observed_at_column/max_staleness_sec handling. // - EQEmu: Spire's own eqemuserver.Client wraps World's telnet `api // get_server_counts`, and Spire exposes it authenticated at // `GET /api/v1/eqemuserver/server-stats` -- there is no DB-backed count // (peq has no online-tracking table, verified against the live schema), -// so this is read via a dedicated Spire service account + JWT, not mysql. +// so this is read via a dedicated Spire service account +// (`spire user:create drop-service ...`) + JWT from `POST /auth/login`, +// not mysql. export interface ProbeOutcome { status: "up" | "down" | "unknown"; latencyMs: number | null; @@ -313,12 +319,24 @@ async function probePlayerCount( // shelled-out client. Used by WoW (trinitycore-mysql/characters) and // OpenDAoC (daoc-mysql/opendaoc); see the file header for the exact query // each borrows from that game's own management tooling. + // + // value_column/observed_at_column let a query return a periodic-snapshot + // row (OpenDAoC's serverstats -- written once a minute by the server's own + // StatSave.cs, not on every read) instead of a live aggregate. If + // observed_at_column is set, a row older than max_staleness_sec is treated + // as unavailable rather than presented as a current 0: the server going + // down doesn't stop serverstats from having a last row, so an unbounded + // staleness window would let a truly-dead server keep reporting whatever + // count it had at the moment it died. if (method === "mysql_direct") { const config = playerCountProbe as Record; const host = config.host as string | undefined; const port = Number(config.port ?? 3306); const database = config.database as string | undefined; const query = config.query as string | undefined; + const valueColumn = config.value_column as string | undefined; + const observedAtColumn = config.observed_at_column as string | undefined; + const maxStalenessSec = config.max_staleness_sec as number | undefined; const credsFile = config.credentials_file as string | undefined; const creds = credsFile ? loadCredentialsFile(credsFile) : {}; const user = resolveCredential(config, creds, "user", "user_env_var"); @@ -348,7 +366,33 @@ async function probePlayerCount( }); const [rows] = await conn.query(query); const row = rows[0] as Record | undefined; - const raw = row ? Object.values(row)[0] : undefined; + if (!row) { + return { + playersOnline: null, + note: "mysql_direct query returned no rows", + }; + } + + if (observedAtColumn && maxStalenessSec) { + const observedAtRaw = row[observedAtColumn]; + const observedAt = + observedAtRaw instanceof Date + ? observedAtRaw + : typeof observedAtRaw === "string" + ? new Date(observedAtRaw) + : null; + const ageSec = observedAt + ? (Date.now() - observedAt.getTime()) / 1000 + : null; + if (ageSec === null || Number.isNaN(ageSec) || ageSec > maxStalenessSec) { + return { + playersOnline: null, + note: `mysql_direct snapshot too stale to trust (${observedAtColumn} missing or older than ${maxStalenessSec}s)`, + }; + } + } + + const raw = valueColumn ? row[valueColumn] : Object.values(row)[0]; const n = typeof raw === "number" ? raw @@ -384,7 +428,15 @@ async function probePlayerCount( if (method === "spire_session_api") { const config = playerCountProbe as Record; const baseUrl = config.base_url as string | undefined; - const loginPath = (config.login_path as string | undefined) ?? "/api/v1/login"; + // NOT under /api/v1 -- Spire's auth controller group is mounted at the + // bare /auth prefix (confirmed live via Spire's own unauthenticated + // GET /api/v1/routes introspection endpoint, which lists + // {"method":"POST","path":"/auth/login"}). Guessing /api/v1/login or + // /api/v1/auth/login both 404/405 -- this is not a documented contract, + // so if the login path ever changes, /api/v1/routes is the ground truth + // to re-check, not the swagger @BasePath annotation on http.go (which + // only covers the OTHER controller groups, not auth's). + const loginPath = (config.login_path as string | undefined) ?? "/auth/login"; const statsPath = (config.stats_path as string | undefined) ?? "/api/v1/eqemuserver/server-stats";