From 2fe19c39e40fc350af39f72e1a3bac28f9ce9874 Mon Sep 17 00:00:00 2001 From: gglucass Date: Thu, 2 Jul 2026 23:25:24 +0200 Subject: [PATCH] feat(stats): surface Codex WS compression counters in /stats summary (#1680) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Codex rides a long-lived WebSocket `/responses` connection. WS units are compressed and counted into the `codex_ws_*` metrics immediately, but turn-level records — the ones that feed `tokens_saved_total` and therefore the `/stats` `summary` block — only land when a `response.completed` frame carries usage tokens. A user watching `summary.api_requests` / `summary.compression` during an active Codex WS session sees frozen counters and concludes Headroom isn't working, even though the `codex_ws` stats section is advancing. (Reported by a Headroom Desktop user who cross-checked `/stats` against a healthy proxy and confirmed-correct Codex routing.) This PR surfaces the live per-unit counters inside `summary` so WS-only sessions are visible at a glance: ```json "codex_ws": {"units_total": 12, "units_modified": 9, "tokens_saved": 4321} ``` The block is deliberately **not** summed into `compression.total_tokens_removed`: turns that did record already contributed the same savings to `tokens_saved_total`, and the recorded-vs-unrecorded split is not tracked globally, so folding the unit sums into the totals would double-count. Additive visibility, not a second ledger. ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [x] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `headroom/proxy/cost.py`: `build_session_summary` emits a `summary.codex_ws` block (`units_total`, `units_modified`, `tokens_saved`) sourced from the live per-unit metrics; only present when `codex_ws_units_total > 0`, so non-Codex sessions keep the existing summary shape. `getattr` defaults keep older/partial metrics objects working. - `tests/test_proxy_dashboard_stats_cache.py`: new `test_session_summary_surfaces_codex_ws_counters`; extended `test_session_summary_uses_generic_cli_filtering_keys` to assert the block is absent when counters are missing. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ uv run --extra dev pytest tests/test_proxy_dashboard_stats_cache.py =================== 11 passed, 1 skipped, 1 warning in 3.47s =================== $ uv run --extra dev pytest tests/test_compression_observability.py tests/test_proxy_healthchecks.py tests/test_pr208_changes.py ======================== 72 passed, 1 warning in 32.18s ======================== $ uv run --extra dev mypy headroom/proxy/cost.py Success: no issues found in 1 source file $ ruff check headroom/proxy/cost.py tests/test_proxy_dashboard_stats_cache.py All checks passed! ``` ## Real Behavior Proof - Environment: macOS 15 (Darwin 24.6.0), Python 3.10 venv via `uv`, branch `fix/stats-summary-codex-ws` @ upstream main - Exact command / steps: called `build_session_summary` with metrics carrying `codex_ws_units_total=12`, `codex_ws_units_modified_total=9`, `codex_ws_unit_tokens_saved_sum=4321` (same shape `create_app` passes at `/stats`), printed `summary["codex_ws"]` - Observed result: `{"units_total": 12, "units_modified": 9, "tokens_saved": 4321}`; with counters absent, `"codex_ws" not in summary` - Not tested: end-to-end `/stats` against a live Codex WS session on this build (the installed desktop bundle runs 0.28.0, which predates this branch); unit path is identical since `/stats` calls `build_session_summary` with the live metrics object ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Additional Notes - Documentation / CHANGELOG unchecked: `/stats` response fields aren't documented per-key, and CHANGELOG did not appear to track additive stats fields — happy to add either if maintainers want it. - Follow-up candidate (out of scope here): fold WS savings into the compression *totals* correctly by tracking a `codex_ws_tokens_saved_recorded_total` at turn-record time, so the unrecorded remainder could be added without double-counting. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 --- headroom/proxy/cost.py | 15 +++++++++++ tests/test_proxy_dashboard_stats_cache.py | 33 +++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py index ce91bb04a..45a6937f5 100644 --- a/headroom/proxy/cost.py +++ b/headroom/proxy/cost.py @@ -575,6 +575,21 @@ def build_session_summary( # dropping info the model actually needs). summary["mcp"] = _aggregate_mcp_events() + # Codex WS sessions compress per-unit on the long-lived /responses socket, + # but turn-level records (which feed tokens_saved_total above) only land + # when a response.completed frame carries usage. Surface the live per-unit + # counters so a WS-only session doesn't read as "no activity" mid-turn. + # Kept as a separate block rather than summed into the compression totals: + # turns that DID record already contributed the same savings there, so + # adding the unit sums on top would double-count. + ws_units = getattr(metrics, "codex_ws_units_total", 0) + if ws_units: + summary["codex_ws"] = { + "units_total": ws_units, + "units_modified": getattr(metrics, "codex_ws_units_modified_total", 0), + "tokens_saved": getattr(metrics, "codex_ws_unit_tokens_saved_sum", 0), + } + # Add tip if token mode would help if proxy.config.mode == PROXY_MODE_CACHE and uncompressed_reasons["prefix_frozen"] > 10: summary["tip"] = ( diff --git a/tests/test_proxy_dashboard_stats_cache.py b/tests/test_proxy_dashboard_stats_cache.py index 92056e1ce..ba0cde706 100644 --- a/tests/test_proxy_dashboard_stats_cache.py +++ b/tests/test_proxy_dashboard_stats_cache.py @@ -495,6 +495,39 @@ def test_session_summary_uses_generic_cli_filtering_keys() -> None: assert payload["compression"]["rtk_tokens_avoided"] == 7 assert payload["cost"]["breakdown"]["cli_filtering_savings_usd"] is None assert payload["cost"]["breakdown"]["rtk_savings_usd"] is None + # Metrics fixture has no codex_ws counters -> no codex_ws block. + assert "codex_ws" not in payload + + +def test_session_summary_surfaces_codex_ws_counters() -> None: + from headroom.proxy.cost import build_session_summary + + proxy = SimpleNamespace( + config=SimpleNamespace(mode="token"), + logger=SimpleNamespace(_logs=[]), + cost_tracker=SimpleNamespace(stats=lambda: {}), + ) + metrics = SimpleNamespace( + requests_by_model={}, + tokens_saved_total=0, + codex_ws_units_total=12, + codex_ws_units_modified_total=9, + codex_ws_unit_tokens_saved_sum=4321, + ) + + payload = build_session_summary( + proxy, + metrics, + {}, + cli_tokens_avoided=0, + total_tokens_before=0, + ) + + assert payload["codex_ws"] == { + "units_total": 12, + "units_modified": 9, + "tokens_saved": 4321, + } def test_stats_reset_clears_runtime_proxy_counters(monkeypatch: pytest.MonkeyPatch) -> None: