diff --git a/docker-compose.yml b/docker-compose.yml index 527d03e50..31de7af33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: # if you want to use a custom OpenAI-compatible API endpoint, # uncomment and set the following line with the desired URL # - OPENAI_TARGET_API_URL=https://api.x.ai + # CLI-filtering dashboard figures require the `rtk` binary inside this + # container; it is not installed by this image. See docs/content/docs/docker-install.mdx. ports: - "8787:8787" volumes: diff --git a/docker/docker-compose.native.yml b/docker/docker-compose.native.yml index 916f47d38..b379938e2 100644 --- a/docker/docker-compose.native.yml +++ b/docker/docker-compose.native.yml @@ -13,6 +13,8 @@ services: # the Docker bind-mount source and is intentionally different. HEADROOM_WORKSPACE_DIR: /tmp/headroom-home/.headroom HEADROOM_CONFIG_DIR: /tmp/headroom-home/.headroom/config + # CLI-filtering dashboard figures require the `rtk` binary inside this + # container; it is not installed by this image. See docs/content/docs/docker-install.mdx. volumes: - ${HEADROOM_WORKSPACE:-.}:/workspace - ${HEADROOM_HOST_HOME:?set HEADROOM_HOST_HOME}/.headroom:/tmp/headroom-home/.headroom @@ -33,6 +35,8 @@ services: # above for rationale. HEADROOM_WORKSPACE_DIR: /tmp/headroom-home/.headroom HEADROOM_CONFIG_DIR: /tmp/headroom-home/.headroom/config + # CLI-filtering dashboard figures require the `rtk` binary inside this + # container; it is not installed by this image. See docs/content/docs/docker-install.mdx. ports: - "${HEADROOM_PORT:-8787}:${HEADROOM_PORT:-8787}" volumes: diff --git a/docs/content/docs/docker-install.mdx b/docs/content/docs/docker-install.mdx index b6f351263..d1332800f 100644 --- a/docs/content/docs/docker-install.mdx +++ b/docs/content/docs/docker-install.mdx @@ -191,6 +191,9 @@ That keeps provider auth and runtime config working without maintaining a separa - The install scripts are idempotent: rerunning them refreshes the wrapper and image without duplicating shell profile blocks. - For persistent service and task installs, use the Python-native `headroom install ...` workflow — see [Persistent Installs](/docs/persistent-installs). - For Docker-native `headroom install ...`, the wrapper persists its profile manifest under `~/.headroom/deploy//`. +- The `rtk` binary is not bundled in the Docker image. Dashboard CLI-filtering + savings figures show as "not installed" (not `0`) until `rtk` is installed + inside the container. ## Next steps diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index f8ac9b2ac..1759627b0 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -3056,6 +3056,9 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: ) rtk_tokens_avoided = cli_tokens_avoided if cli_filtering_tool == "rtk" else 0 lean_ctx_tokens_avoided = cli_tokens_avoided if cli_filtering_tool == "lean-ctx" else 0 + cli_filtering_available = bool( + cli_filtering_stats and cli_filtering_stats.get("installed", False) + ) # Calculate total tokens before Headroom-side reduction. Proxy # compression and the configured context tool both remove tokens before @@ -3170,6 +3173,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: "cli_filtering": { "tool": cli_filtering_tool, "label": cli_filtering_label, + "available": cli_filtering_available, "tokens": cli_tokens_avoided, "tokens_saved": cli_tokens_avoided, "session": cli_filtering_session, @@ -3438,9 +3442,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: "context_tool": { "configured": cli_filtering_tool, "label": cli_filtering_label, - "available": bool( - cli_filtering_stats and cli_filtering_stats.get("installed", False) - ), + "available": cli_filtering_available, "stats": cli_filtering_stats, }, "cli_filtering": cli_filtering_stats, @@ -3562,11 +3564,13 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: """Get durable proxy compression history plus display-session state. The JSON payload also carries a ``cli_filtering`` key with live RTK - stats. This is a curated subset (``tool``, ``label``, ``lifetime``, - ``session``) tailored to the Historical tab, not the full - ``_get_context_tool_stats()`` payload that ``/stats`` exposes. It is - ``None`` when RTK is absent or its stats cannot be read, so the tab - simply hides the card rather than erroring. + stats. This is a curated subset (``tool``, ``label``, ``available``, + ``lifetime``, ``session``) tailored to the Historical tab, not the + full ``_get_context_tool_stats()`` payload that ``/stats`` exposes. + It is ``None`` only when the stats read hard-fails; when the tool is + merely absent, ``cli_filtering`` stays populated with + ``available: False`` and zeroed counters so the tab can distinguish + "not installed" from "installed, no data yet." """ if format == "csv": filename = f"headroom-stats-history-{series}.csv" @@ -3593,6 +3597,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: history["cli_filtering"] = { "tool": str(cli_stats.get("tool", "rtk")), "label": str(cli_stats.get("label", "RTK")), + "available": bool(cli_stats.get("installed", False)), "lifetime": cli_stats.get("lifetime", {}), "session": cli_stats.get("session", {}), } diff --git a/tests/test_dashboard_context_tool_availability_playwright.py b/tests/test_dashboard_context_tool_availability_playwright.py new file mode 100644 index 000000000..90ce9085c --- /dev/null +++ b/tests/test_dashboard_context_tool_availability_playwright.py @@ -0,0 +1,233 @@ +"""Behavior-driven Playwright validation for context-tool (RTK) availability +messaging on the dashboard (issue #1831). + +Local runs only import/collect this file (Windows dev sandboxes don't run the +real browser here); the "Dashboard Playwright" CI check executes it for real. +""" + +from __future__ import annotations + +import copy +import json +from urllib.parse import urlsplit + +import pytest + +from headroom.dashboard import get_dashboard_html + +playwright = pytest.importorskip("playwright.sync_api") +Page = playwright.Page +expect = playwright.expect +sync_playwright = playwright.sync_playwright + + +def _base_stats() -> dict: + return { + "cost": { + "savings_usd": 12.34, + "compression_savings_usd": 12.34, + "cache_savings_usd": 5.25, + "cli_tokens_avoided": 0, + }, + "requests": { + "total": 128, + "cached": 96, + "rate_limited": 0, + "failed": 0, + "by_provider": {"anthropic": 128}, + "by_model": {"claude-opus-4-6": 128}, + }, + "tokens": { + "input": 245_000, + "output": 88_000, + "saved": 143_000, + "cli_tokens_avoided": 0, + "total_before_compression": 388_000, + "savings_percent": 36.86, + }, + "overhead": {"average_ms": 14.2, "min_ms": 4.5, "max_ms": 42.7}, + "ttfb": {"average_ms": 1320.0, "min_ms": 420.0, "max_ms": 2900.0}, + "latency": {"average_ms": 1510.0, "min_ms": 520.0, "max_ms": 3300.0}, + "waste_signals": {"json_bloat": 95_000, "repetition": 48_000}, + "savings_history": [ + ["2026-04-01T00:00:00Z", 12_000], + ["2026-04-05T00:00:00Z", 143_000], + ], + "persistent_savings": { + "display_session": {}, + "lifetime": {"tokens_saved": 143_000, "compression_savings_usd": 12.34}, + }, + "pipeline_timing": {}, + "compression_cache": {"mode": "cache"}, + "prefix_cache": {"by_provider": {}, "totals": {}, "prefix_freeze": {}}, + } + + +def _sample_stats(*, available: bool, tokens_saved: int = 0) -> dict: + """Build a /stats payload with context_tool/cli_filtering availability set. + + `_base_stats()` has no `savings` key at all, so writing + `savings.by_layer.cli_filtering` requires a `.setdefault(...)` chain + rather than direct key assignment (would otherwise raise `KeyError`). + """ + stats = copy.deepcopy(_base_stats()) + stats["tokens"]["cli_tokens_avoided"] = tokens_saved + stats["context_tool"] = { + "configured": "rtk", + "label": "RTK", + "available": available, + "stats": {"tool": "rtk", "label": "RTK", "installed": available}, + } + cli_filtering = ( + stats.setdefault("savings", {}).setdefault("by_layer", {}).setdefault("cli_filtering", {}) + ) + cli_filtering.update( + { + "tool": "rtk", + "label": "RTK", + "available": available, + "tokens": tokens_saved, + "tokens_saved": tokens_saved, + "session": {}, + "lifetime": {"tokens_saved": 0}, + "session_savings_pct": 0.0, + } + ) + return stats + + +def _sample_history(*, available: bool, lifetime_tokens_saved: int = 456_700) -> dict: + return { + "history": [ + { + "timestamp": "2026-04-05T00:00:00Z", + "total_tokens_saved": 143_000, + "compression_savings_usd": 12.34, + }, + ], + "series": {"daily": [], "weekly": [], "monthly": []}, + "lifetime": {"tokens_saved": 143_000, "compression_savings_usd": 12.34}, + "cli_filtering": { + "tool": "rtk", + "label": "RTK", + "available": available, + "lifetime": {"tokens_saved": lifetime_tokens_saved}, + "session": {}, + }, + } + + +def _install_dashboard_routes(page: Page, stats: dict, history: dict) -> None: + health = {"status": "healthy", "version": "0.3.0"} + dashboard_html = get_dashboard_html() + + def handler(route) -> None: # type: ignore[no-untyped-def] + # Match on the URL path only: the dashboard fetches /stats?cached=1, + # so suffix checks against the full URL miss it and the request + # escapes the harness to the real network. + path = urlsplit(route.request.url).path + if path in ("/dashboard", "/"): + route.fulfill(status=200, content_type="text/html", body=dashboard_html) + return + if "/stats-history" in path: + route.fulfill( + status=200, + content_type="application/json", + body=json.dumps(history), + ) + return + if path.endswith("/stats"): + route.fulfill(status=200, content_type="application/json", body=json.dumps(stats)) + return + if path.endswith("/health"): + route.fulfill(status=200, content_type="application/json", body=json.dumps(health)) + return + route.continue_() + + page.route("**/*", handler) + + +def test_dashboard_session_view_shows_not_installed_message_when_unavailable() -> None: + """Session view shows a distinct "not installed" message, not `0`, when + `context_tool.available` is False (the #1831 bug this fix addresses). + """ + stats = _sample_stats(available=False, tokens_saved=0) + history = _sample_history(available=False) + + with sync_playwright() as pw: + browser = pw.chromium.launch() + page = browser.new_page(viewport={"width": 1720, "height": 1400}, color_scheme="dark") + _install_dashboard_routes(page, stats, history) + page.goto("http://headroom.local/dashboard", wait_until="load") + + expect(page.get_by_text("RTK not installed", exact=True)).to_be_visible() + expect(page.get_by_text("not installed", exact=True)).to_be_visible() + expect(page.get_by_text("RTK 0 this session (0.0%)", exact=True)).to_have_count(0) + + browser.close() + + +def test_dashboard_session_view_shows_real_zero_row_when_installed_but_zero() -> None: + """Boundary value 0.0: installed but genuinely zero savings still renders + the real number, not the "not installed" message -- proves the new guard + doesn't over-trigger on the exact case the #1831 reporter would hit again. + """ + stats = _sample_stats(available=True, tokens_saved=0) + history = _sample_history(available=True, lifetime_tokens_saved=0) + + with sync_playwright() as pw: + browser = pw.chromium.launch() + page = browser.new_page(viewport={"width": 1720, "height": 1400}, color_scheme="dark") + _install_dashboard_routes(page, stats, history) + page.goto("http://headroom.local/dashboard", wait_until="load") + + expect(page.get_by_text("RTK 0 this session (0.0%)", exact=True)).to_be_visible() + expect(page.get_by_text("RTK not installed", exact=True)).to_have_count(0) + # The Token Usage panel's "not installed" row uses `x-show`, which + # toggles CSS display and keeps the node in the DOM (unlike the + # ternary-swapped "RTK not installed" text above, which is genuinely + # absent). Assert hidden, not absent, matching the repo's existing + # `x-show` convention in tests/test_dashboard_cache_lifetime_playwright.py. + expect(page.get_by_text("not installed", exact=True)).to_be_hidden() + + browser.close() + + +def test_dashboard_historical_tab_hides_lifetime_card_when_unavailable() -> None: + """The Historical tab's lifetime card stays hidden (same as the existing + hard-failure hide-card behavior) when `cli_filtering.available` is False. + """ + stats = _sample_stats(available=True, tokens_saved=5_000) + history = _sample_history(available=False) + + with sync_playwright() as pw: + browser = pw.chromium.launch() + page = browser.new_page(viewport={"width": 1720, "height": 1400}, color_scheme="dark") + _install_dashboard_routes(page, stats, history) + page.goto("http://headroom.local/dashboard", wait_until="load") + + page.get_by_role("button", name="Historical").click() + expect(page.get_by_text("Historical Summary")).to_be_visible() + expect(page.get_by_text("RTK Lifetime Saved")).to_have_count(0) + + browser.close() + + +def test_dashboard_historical_tab_shows_lifetime_card_when_available() -> None: + """The Historical tab's lifetime card still renders the real number when + `cli_filtering.available` is True (existing behavior, unchanged). + """ + stats = _sample_stats(available=True, tokens_saved=5_000) + history = _sample_history(available=True, lifetime_tokens_saved=456_700) + + with sync_playwright() as pw: + browser = pw.chromium.launch() + page = browser.new_page(viewport={"width": 1720, "height": 1400}, color_scheme="dark") + _install_dashboard_routes(page, stats, history) + page.goto("http://headroom.local/dashboard", wait_until="load") + + page.get_by_role("button", name="Historical").click() + expect(page.get_by_text("RTK Lifetime Saved")).to_be_visible() + expect(page.get_by_text("456.7k")).to_be_visible() + + browser.close() diff --git a/tests/test_proxy_dashboard_stats_cache.py b/tests/test_proxy_dashboard_stats_cache.py index 5a0cc0764..ece4c0ef7 100644 --- a/tests/test_proxy_dashboard_stats_cache.py +++ b/tests/test_proxy_dashboard_stats_cache.py @@ -445,6 +445,138 @@ def test_stats_reports_lean_ctx_as_selected_cli_filter(monkeypatch: pytest.Monke assert payload["savings"]["by_layer"]["compression"]["lean_ctx_tokens"] == 9 +def test_stats_cli_filtering_available_false_when_not_installed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reproduction: savings.by_layer.cli_filtering.available reflects `installed` + when the context tool isn't installed. On origin/main, `available` doesn't + exist in this dict at all (`KeyError`); this asserts the fixed key/value. + """ + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + import headroom.proxy.server as server + from headroom.proxy.server import ProxyConfig, create_app + + monkeypatch.setattr( + server, + "get_compression_store", + lambda: _StatsStub({"store": 0}, "store", {}), + ) + monkeypatch.setattr( + server, + "get_telemetry_collector", + lambda: _StatsStub({"telemetry": 0}, "telemetry", {}), + ) + monkeypatch.setattr( + server, + "get_compression_feedback", + lambda: _StatsStub({"feedback": 0}, "feedback", {}), + ) + monkeypatch.setattr( + server, + "_get_context_tool_stats", + lambda: { + "tool": "rtk", + "label": "RTK", + "installed": False, + "total_commands": 0, + "tokens_saved": 0, + "avg_savings_pct": 0.0, + }, + ) + monkeypatch.setattr(server, "get_toin", lambda: _ToinStub()) + + app = create_app( + ProxyConfig( + optimize=False, + cache_enabled=False, + rate_limit_enabled=False, + cost_tracking_enabled=False, + log_requests=False, + ccr_inject_tool=False, + ccr_handle_responses=False, + ccr_context_tracking=False, + ) + ) + + with TestClient(app) as client: + response = client.get("/stats") + + payload = response.json() + assert response.status_code == 200 + assert payload["savings"]["by_layer"]["cli_filtering"]["available"] is False + # Preservation: context_tool.available keeps matching the same `installed` + # value it always did, now computed via the hoisted local. + assert payload["context_tool"]["available"] is False + + +def test_stats_cli_filtering_available_true_at_boundary_zero_tokens_saved( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Boundary value 0.0: installed but genuinely zero savings must still + report `available: True` with a real `0`, never collapsing into the + "not installed" state. This is the negative-space guard against the fix + over-triggering on the #1831 reporter's original zero-figures symptom. + """ + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + import headroom.proxy.server as server + from headroom.proxy.server import ProxyConfig, create_app + + monkeypatch.setattr( + server, + "get_compression_store", + lambda: _StatsStub({"store": 0}, "store", {}), + ) + monkeypatch.setattr( + server, + "get_telemetry_collector", + lambda: _StatsStub({"telemetry": 0}, "telemetry", {}), + ) + monkeypatch.setattr( + server, + "get_compression_feedback", + lambda: _StatsStub({"feedback": 0}, "feedback", {}), + ) + monkeypatch.setattr( + server, + "_get_context_tool_stats", + lambda: { + "tool": "rtk", + "label": "RTK", + "installed": True, + "total_commands": 0, + "tokens_saved": 0, + "avg_savings_pct": 0.0, + }, + ) + monkeypatch.setattr(server, "get_toin", lambda: _ToinStub()) + + app = create_app( + ProxyConfig( + optimize=False, + cache_enabled=False, + rate_limit_enabled=False, + cost_tracking_enabled=False, + log_requests=False, + ccr_inject_tool=False, + ccr_handle_responses=False, + ccr_context_tracking=False, + ) + ) + + with TestClient(app) as client: + response = client.get("/stats") + + payload = response.json() + assert response.status_code == 200 + assert payload["savings"]["by_layer"]["cli_filtering"]["available"] is True + assert payload["savings"]["by_layer"]["cli_filtering"]["tokens_saved"] == 0 + assert payload["context_tool"]["available"] is True + + def test_cost_merge_uses_generic_cli_filtering_name() -> None: from headroom.proxy.cost import merge_cost_stats diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py index 791e01fb7..7e353d85a 100644 --- a/tests/test_proxy_savings_history.py +++ b/tests/test_proxy_savings_history.py @@ -1380,6 +1380,83 @@ def test_stats_history_includes_cli_filtering(tmp_path, monkeypatch): assert data["cli_filtering"]["lifetime"]["tokens_saved"] == 999 +def test_stats_history_cli_filtering_available_false_when_not_installed(tmp_path, monkeypatch): + """Reproduction: /stats-history's curated cli_filtering block must carry + `available` reflecting the backend `installed` flag. On origin/main this + key doesn't exist in the curated dict at all (`KeyError`); this asserts + the fixed key/value. The tool being merely absent must NOT collapse the + block to `None` -- it stays populated with `available: False` and zeroed + counters so the Historical tab can distinguish absence from a hard + read failure. + """ + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + import headroom.proxy.server as server + from headroom.proxy.server import ProxyConfig, create_app + + savings_path = tmp_path / "proxy_savings.json" + monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path)) + + _rtk_not_installed_payload = { + "tool": "rtk", + "label": "RTK", + "installed": False, + "tokens_saved": 0, + "session": {"tokens_saved": 0, "commands": 0}, + "lifetime": {"tokens_saved": 0, "commands": 0}, + } + monkeypatch.setattr(server, "_get_context_tool_stats", lambda: _rtk_not_installed_payload) + + config = ProxyConfig( + cache_enabled=False, + rate_limit_enabled=False, + log_requests=False, + ) + + with TestClient(create_app(config)) as client: + response = client.get("/stats-history") + assert response.status_code == 200 + data = response.json() + + assert data["cli_filtering"] is not None + assert data["cli_filtering"]["available"] is False + + +def test_stats_history_cli_filtering_stays_none_on_hard_read_failure(tmp_path, monkeypatch): + """Preservation: /stats-history's cli_filtering key stays `None` only when + the underlying stats read hard-fails (exception), not merely because the + tool is absent -- the Historical tab keeps hiding the card in that case, + unchanged from prior behavior. + """ + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + import headroom.proxy.server as server + from headroom.proxy.server import ProxyConfig, create_app + + savings_path = tmp_path / "proxy_savings.json" + monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path)) + + def _raise() -> dict: + raise RuntimeError("simulated hard stats-read failure") + + monkeypatch.setattr(server, "_get_context_tool_stats", _raise) + + config = ProxyConfig( + cache_enabled=False, + rate_limit_enabled=False, + log_requests=False, + ) + + with TestClient(create_app(config)) as client: + response = client.get("/stats-history") + assert response.status_code == 200 + data = response.json() + + assert data["cli_filtering"] is None + + def test_coercion_helpers_reject_non_finite_values(): """Non-finite inputs fail open to the default -- never raise, never leak NaN/inf.