From 53a465b121e0a7f45f862a21829639423226a5eb Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Wed, 8 Jul 2026 00:24:41 -0400 Subject: [PATCH] fix(proxy): subtract cache write premiums from net savings (#1800) ## Description Cache stats already calculate both prompt-cache read savings and cache-write premium cost, but the exported `net_savings_usd` field used gross read savings alone. That made cache-heavy token-mode workloads look profitable even when extra cache writes offset or exceeded the read discount. This updates existing cache cost accounting so provider and total `net_savings_usd` subtract write premiums while keeping gross savings and write premium fields visible. Refs #327. The scope follows doublefx's controlled measurement in https://github.com/headroomlabs-ai/headroom/issues/327#issuecomment-4683604089, which showed token-mode compression increasing cache write volume and billed cost while dashboard token savings looked positive. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Subtract cache write premiums from provider-level cache `net_savings_usd`. - Subtract aggregate cache write premiums from total cache `net_savings_usd`. - Keep gross `savings_usd` and `write_premium_usd` visible for dashboard and telemetry consumers. - Add focused regressions for provider net, total net, and zero-write-premium preservation. - Update the dashboard cache TTL fixture to match the corrected net value. ## Testing - [x] Unit tests pass (`uv run pytest tests/test_proxy_cache_ttl_metrics.py tests/test_dashboard_cache_ttl_playwright.py tests/test_proxy_dashboard_stats_cache.py -q`) - [x] Linting passes (`uv run ruff check headroom/proxy/cost.py tests/test_proxy_cache_ttl_metrics.py tests/test_dashboard_cache_ttl_playwright.py tests/test_proxy_dashboard_stats_cache.py`) - [ ] Type checking passes (`uv run mypy headroom`) - [x] New tests added for new functionality when applicable - [x] Manual testing performed ### Test Output ```text uv run pytest tests/test_proxy_cache_ttl_metrics.py tests/test_dashboard_cache_ttl_playwright.py tests/test_proxy_dashboard_stats_cache.py -q 28 passed, 2 skipped, 1 warning in 32.75s uv run pytest tests/test_proxy_cache_ttl_metrics.py -q -k keeps_net_equal_without_write_premium 1 passed, 16 deselected in 0.15s uv run ruff check headroom/proxy/cost.py tests/test_proxy_cache_ttl_metrics.py tests/test_dashboard_cache_ttl_playwright.py tests/test_proxy_dashboard_stats_cache.py All checks passed! ``` ## Real Behavior Proof - Environment: Windows, Python through the project `uv` environment. - Exact command / steps: run the cache net-savings regressions against base and head. - Observed result: base reports provider net as `0.0036` instead of `0.0021` and total net as `0.0046` instead of `0.0031`; head passes the focused cache metrics suite and preserves `net_savings_usd == savings_usd` when there is no write premium. - Not tested: broader cache-hit-rate tuning, prompt-cache policy changes, and live provider billing. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I have updated the CHANGELOG.md if applicable ## Additional Notes No changelog entry is needed because this corrects existing stats fields rather than adding a new command or control. Type checking was not part of the focused local validation for this Python-only fix. Dashboard Playwright coverage is CI-owned locally; the import-gated file was included in the focused pytest command and skipped because Playwright is not installed in this environment. --- headroom/proxy/cost.py | 11 +- tests/test_dashboard_cache_ttl_playwright.py | 6 +- tests/test_proxy_cache_ttl_metrics.py | 104 +++++++++++++++++++ 3 files changed, 112 insertions(+), 9 deletions(-) diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py index 45a6937f5..c419d4cbc 100644 --- a/headroom/proxy/cost.py +++ b/headroom/proxy/cost.py @@ -159,9 +159,8 @@ def build_prefix_cache_stats( # Calculate savings: # Cache reads save (1.0 - read_mult) per token vs uncached input price. - # Cache write premium is NOT deducted — it's baseline cost that the - # client (e.g. Claude Code) pays regardless of Headroom. We track it - # for observability but don't penalise our savings number. + # Cache write premium stays visible as its own gross field, and net + # savings subtract it so the dashboard reflects billed cache impact. read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment] write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment] write_5m_tokens: int = pc["cache_write_5m_tokens"] # type: ignore[assignment] @@ -174,7 +173,7 @@ def build_prefix_cache_stats( if input_price_per_token: # Savings from reads: tokens * price * (1.0 - read_multiplier) savings_usd = read_tokens * input_price_per_token * (1.0 - read_mult) - # Write premium (observability only — not subtracted from savings) + # Write premium is reported separately and subtracted from net savings. if write_mult > 1.0: write_premium_usd = write_tokens * input_price_per_token * (write_mult - 1.0) @@ -205,7 +204,7 @@ def build_prefix_cache_stats( "write_premium": f"{(write_mult - 1.0) * 100:.0f}%" if write_mult > 1.0 else "none", "savings_usd": round(savings_usd, 4), "write_premium_usd": round(write_premium_usd, 4), - "net_savings_usd": round(savings_usd, 4), + "net_savings_usd": round(savings_usd - write_premium_usd, 4), "label": str(econ["label"]), "observed_ttl_buckets": { "5m": { @@ -246,7 +245,7 @@ def build_prefix_cache_stats( totals["savings_usd"] += savings_usd totals["write_premium_usd"] += write_premium_usd - totals["net_savings_usd"] = round(totals["savings_usd"], 4) + totals["net_savings_usd"] = round(totals["savings_usd"] - totals["write_premium_usd"], 4) totals["savings_usd"] = round(totals["savings_usd"], 4) totals["write_premium_usd"] = round(totals["write_premium_usd"], 4) # Token-level hit rate across all providers diff --git a/tests/test_dashboard_cache_ttl_playwright.py b/tests/test_dashboard_cache_ttl_playwright.py index a2ae2ad88..3d8959eb8 100644 --- a/tests/test_dashboard_cache_ttl_playwright.py +++ b/tests/test_dashboard_cache_ttl_playwright.py @@ -22,7 +22,7 @@ def _sample_stats() -> dict: "cost": { "savings_usd": 12.34, "compression_savings_usd": 12.34, - "cache_savings_usd": 5.67, + "cache_savings_usd": 5.25, "cli_tokens_avoided": 0, }, "requests": { @@ -76,7 +76,7 @@ def _sample_stats() -> dict: "write_premium": "25%", "savings_usd": 5.67, "write_premium_usd": 0.42, - "net_savings_usd": 5.67, + "net_savings_usd": 5.25, "label": "Explicit breakpoints, 5-min TTL", "observed_ttl_buckets": { "5m": {"tokens": 185_000, "requests": 18}, @@ -102,7 +102,7 @@ def _sample_stats() -> dict: "bust_write_tokens": 0, "savings_usd": 5.67, "write_premium_usd": 0.42, - "net_savings_usd": 5.67, + "net_savings_usd": 5.25, "hit_rate": 75.0, "observed_ttl_buckets": { "5m": {"tokens": 185_000, "requests": 18}, diff --git a/tests/test_proxy_cache_ttl_metrics.py b/tests/test_proxy_cache_ttl_metrics.py index e603ddb5b..35ab3e96a 100644 --- a/tests/test_proxy_cache_ttl_metrics.py +++ b/tests/test_proxy_cache_ttl_metrics.py @@ -80,6 +80,110 @@ def test_prefix_cache_stats_include_observed_ttl_mix() -> None: assert stats["totals"]["observed_ttl_buckets"]["1h"]["tokens"] == 45 +def test_prefix_cache_stats_subtracts_write_premium_from_provider_net_savings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + metrics = PrometheusMetrics() + metrics.cache_by_provider["anthropic"].update( + { + "requests": 2, + "hit_requests": 1, + "cache_read_tokens": 40, + "cache_write_tokens": 60, + "cache_write_5m_tokens": 60, + "cache_write_1h_tokens": 0, + "cache_write_5m_requests": 1, + "cache_write_1h_requests": 0, + } + ) + + tracker = CostTracker() + tracker._tokens_sent_by_model.update({"claude-opus-4-6": 1}) + monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 100.0) + + stats = build_prefix_cache_stats(metrics, tracker) + + anthropic = stats["by_provider"]["anthropic"] + + assert anthropic["savings_usd"] == 0.0036 + assert anthropic["write_premium_usd"] == 0.0015 + assert anthropic["net_savings_usd"] == 0.0021 + + +def test_prefix_cache_stats_subtracts_write_premium_from_total_net_savings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + metrics = PrometheusMetrics() + metrics.cache_by_provider["anthropic"].update( + { + "requests": 2, + "hit_requests": 1, + "cache_read_tokens": 40, + "cache_write_tokens": 60, + "cache_write_5m_tokens": 60, + "cache_write_1h_tokens": 0, + "cache_write_5m_requests": 1, + "cache_write_1h_requests": 0, + } + ) + metrics.cache_by_provider["openai"].update( + { + "requests": 1, + "hit_requests": 1, + "cache_read_tokens": 20, + "cache_write_tokens": 10, + "cache_write_5m_tokens": 0, + "cache_write_1h_tokens": 10, + "cache_write_5m_requests": 0, + "cache_write_1h_requests": 1, + } + ) + + tracker = CostTracker() + tracker._tokens_sent_by_model.update({"claude-opus-4-6": 1, "gpt-4o": 1}) + monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 100.0) + + stats = build_prefix_cache_stats(metrics, tracker) + + openai = stats["by_provider"]["openai"] + + assert openai["write_premium_usd"] == 0.0 + assert openai["net_savings_usd"] == openai["savings_usd"] + assert stats["totals"]["savings_usd"] == 0.0046 + assert stats["totals"]["write_premium_usd"] == 0.0015 + assert stats["totals"]["net_savings_usd"] == 0.0031 + + +def test_prefix_cache_stats_keeps_net_equal_without_write_premium( + monkeypatch: pytest.MonkeyPatch, +) -> None: + metrics = PrometheusMetrics() + metrics.cache_by_provider["openai"].update( + { + "requests": 1, + "hit_requests": 1, + "cache_read_tokens": 20, + "cache_write_tokens": 0, + "cache_write_5m_tokens": 0, + "cache_write_1h_tokens": 0, + "cache_write_5m_requests": 0, + "cache_write_1h_requests": 0, + } + ) + + tracker = CostTracker() + tracker._tokens_sent_by_model.update({"gpt-4o": 1}) + monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 100.0) + + stats = build_prefix_cache_stats(metrics, tracker) + + openai = stats["by_provider"]["openai"] + + assert openai["savings_usd"] == 0.001 + assert openai["write_premium_usd"] == 0.0 + assert openai["net_savings_usd"] == openai["savings_usd"] + + def test_prometheus_metrics_export_includes_extended_fields(tmp_path) -> None: metrics = PrometheusMetrics( savings_tracker=SavingsTracker(path=str(tmp_path / "proxy_savings.json"))