fix(proxy): subtract cache write premiums from net savings (#1800)

## Description

Cache stats already calculate both prompt-cache read savings and
cache-write premium cost, but the exported `net_savings_usd` field used
gross read savings alone. That made cache-heavy token-mode workloads
look profitable even when extra cache writes offset or exceeded the read
discount. This updates existing cache cost accounting so provider and
total `net_savings_usd` subtract write premiums while keeping gross
savings and write premium fields visible. Refs #327.

The scope follows doublefx's controlled measurement in
https://github.com/headroomlabs-ai/headroom/issues/327#issuecomment-4683604089,
which showed token-mode compression increasing cache write volume and
billed cost while dashboard token savings looked positive.

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)

## Changes Made

- Subtract cache write premiums from provider-level cache
`net_savings_usd`.
- Subtract aggregate cache write premiums from total cache
`net_savings_usd`.
- Keep gross `savings_usd` and `write_premium_usd` visible for dashboard
and telemetry consumers.
- Add focused regressions for provider net, total net, and
zero-write-premium preservation.
- Update the dashboard cache TTL fixture to match the corrected net
value.

## Testing

- [x] Unit tests pass (`uv run pytest
tests/test_proxy_cache_ttl_metrics.py
tests/test_dashboard_cache_ttl_playwright.py
tests/test_proxy_dashboard_stats_cache.py -q`)
- [x] Linting passes (`uv run ruff check headroom/proxy/cost.py
tests/test_proxy_cache_ttl_metrics.py
tests/test_dashboard_cache_ttl_playwright.py
tests/test_proxy_dashboard_stats_cache.py`)
- [ ] Type checking passes (`uv run mypy headroom`)
- [x] New tests added for new functionality when applicable
- [x] Manual testing performed

### Test Output

```text
uv run pytest tests/test_proxy_cache_ttl_metrics.py tests/test_dashboard_cache_ttl_playwright.py tests/test_proxy_dashboard_stats_cache.py -q
28 passed, 2 skipped, 1 warning in 32.75s

uv run pytest tests/test_proxy_cache_ttl_metrics.py -q -k keeps_net_equal_without_write_premium
1 passed, 16 deselected in 0.15s

uv run ruff check headroom/proxy/cost.py tests/test_proxy_cache_ttl_metrics.py tests/test_dashboard_cache_ttl_playwright.py tests/test_proxy_dashboard_stats_cache.py
All checks passed!
```

## Real Behavior Proof

- Environment: Windows, Python through the project `uv` environment.
- Exact command / steps: run the cache net-savings regressions against
base and head.
- Observed result: base reports provider net as `0.0036` instead of
`0.0021` and total net as `0.0046` instead of `0.0031`; head passes the
focused cache metrics suite and preserves `net_savings_usd ==
savings_usd` when there is no write premium.
- Not tested: broader cache-hit-rate tuning, prompt-cache policy
changes, and live provider billing.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] I have updated the CHANGELOG.md if applicable

## Additional Notes

No changelog entry is needed because this corrects existing stats fields
rather than adding a new command or control. Type checking was not part
of the focused local validation for this Python-only fix. Dashboard
Playwright coverage is CI-owned locally; the import-gated file was
included in the focused pytest command and skipped because Playwright is
not installed in this environment.
This commit is contained in:
Rod Boev 2026-07-08 00:24:41 -04:00 committed by GitHub
parent 931eed879d
commit 53a465b121
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 112 additions and 9 deletions

View file

@ -159,9 +159,8 @@ def build_prefix_cache_stats(
# Calculate savings:
# Cache reads save (1.0 - read_mult) per token vs uncached input price.
# Cache write premium is NOT deducted — it's baseline cost that the
# client (e.g. Claude Code) pays regardless of Headroom. We track it
# for observability but don't penalise our savings number.
# Cache write premium stays visible as its own gross field, and net
# savings subtract it so the dashboard reflects billed cache impact.
read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment]
write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment]
write_5m_tokens: int = pc["cache_write_5m_tokens"] # type: ignore[assignment]
@ -174,7 +173,7 @@ def build_prefix_cache_stats(
if input_price_per_token:
# Savings from reads: tokens * price * (1.0 - read_multiplier)
savings_usd = read_tokens * input_price_per_token * (1.0 - read_mult)
# Write premium (observability only — not subtracted from savings)
# Write premium is reported separately and subtracted from net savings.
if write_mult > 1.0:
write_premium_usd = write_tokens * input_price_per_token * (write_mult - 1.0)
@ -205,7 +204,7 @@ def build_prefix_cache_stats(
"write_premium": f"{(write_mult - 1.0) * 100:.0f}%" if write_mult > 1.0 else "none",
"savings_usd": round(savings_usd, 4),
"write_premium_usd": round(write_premium_usd, 4),
"net_savings_usd": round(savings_usd, 4),
"net_savings_usd": round(savings_usd - write_premium_usd, 4),
"label": str(econ["label"]),
"observed_ttl_buckets": {
"5m": {
@ -246,7 +245,7 @@ def build_prefix_cache_stats(
totals["savings_usd"] += savings_usd
totals["write_premium_usd"] += write_premium_usd
totals["net_savings_usd"] = round(totals["savings_usd"], 4)
totals["net_savings_usd"] = round(totals["savings_usd"] - totals["write_premium_usd"], 4)
totals["savings_usd"] = round(totals["savings_usd"], 4)
totals["write_premium_usd"] = round(totals["write_premium_usd"], 4)
# Token-level hit rate across all providers

View file

@ -22,7 +22,7 @@ def _sample_stats() -> dict:
"cost": {
"savings_usd": 12.34,
"compression_savings_usd": 12.34,
"cache_savings_usd": 5.67,
"cache_savings_usd": 5.25,
"cli_tokens_avoided": 0,
},
"requests": {
@ -76,7 +76,7 @@ def _sample_stats() -> dict:
"write_premium": "25%",
"savings_usd": 5.67,
"write_premium_usd": 0.42,
"net_savings_usd": 5.67,
"net_savings_usd": 5.25,
"label": "Explicit breakpoints, 5-min TTL",
"observed_ttl_buckets": {
"5m": {"tokens": 185_000, "requests": 18},
@ -102,7 +102,7 @@ def _sample_stats() -> dict:
"bust_write_tokens": 0,
"savings_usd": 5.67,
"write_premium_usd": 0.42,
"net_savings_usd": 5.67,
"net_savings_usd": 5.25,
"hit_rate": 75.0,
"observed_ttl_buckets": {
"5m": {"tokens": 185_000, "requests": 18},

View file

@ -80,6 +80,110 @@ def test_prefix_cache_stats_include_observed_ttl_mix() -> None:
assert stats["totals"]["observed_ttl_buckets"]["1h"]["tokens"] == 45
def test_prefix_cache_stats_subtracts_write_premium_from_provider_net_savings(
monkeypatch: pytest.MonkeyPatch,
) -> None:
metrics = PrometheusMetrics()
metrics.cache_by_provider["anthropic"].update(
{
"requests": 2,
"hit_requests": 1,
"cache_read_tokens": 40,
"cache_write_tokens": 60,
"cache_write_5m_tokens": 60,
"cache_write_1h_tokens": 0,
"cache_write_5m_requests": 1,
"cache_write_1h_requests": 0,
}
)
tracker = CostTracker()
tracker._tokens_sent_by_model.update({"claude-opus-4-6": 1})
monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 100.0)
stats = build_prefix_cache_stats(metrics, tracker)
anthropic = stats["by_provider"]["anthropic"]
assert anthropic["savings_usd"] == 0.0036
assert anthropic["write_premium_usd"] == 0.0015
assert anthropic["net_savings_usd"] == 0.0021
def test_prefix_cache_stats_subtracts_write_premium_from_total_net_savings(
monkeypatch: pytest.MonkeyPatch,
) -> None:
metrics = PrometheusMetrics()
metrics.cache_by_provider["anthropic"].update(
{
"requests": 2,
"hit_requests": 1,
"cache_read_tokens": 40,
"cache_write_tokens": 60,
"cache_write_5m_tokens": 60,
"cache_write_1h_tokens": 0,
"cache_write_5m_requests": 1,
"cache_write_1h_requests": 0,
}
)
metrics.cache_by_provider["openai"].update(
{
"requests": 1,
"hit_requests": 1,
"cache_read_tokens": 20,
"cache_write_tokens": 10,
"cache_write_5m_tokens": 0,
"cache_write_1h_tokens": 10,
"cache_write_5m_requests": 0,
"cache_write_1h_requests": 1,
}
)
tracker = CostTracker()
tracker._tokens_sent_by_model.update({"claude-opus-4-6": 1, "gpt-4o": 1})
monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 100.0)
stats = build_prefix_cache_stats(metrics, tracker)
openai = stats["by_provider"]["openai"]
assert openai["write_premium_usd"] == 0.0
assert openai["net_savings_usd"] == openai["savings_usd"]
assert stats["totals"]["savings_usd"] == 0.0046
assert stats["totals"]["write_premium_usd"] == 0.0015
assert stats["totals"]["net_savings_usd"] == 0.0031
def test_prefix_cache_stats_keeps_net_equal_without_write_premium(
monkeypatch: pytest.MonkeyPatch,
) -> None:
metrics = PrometheusMetrics()
metrics.cache_by_provider["openai"].update(
{
"requests": 1,
"hit_requests": 1,
"cache_read_tokens": 20,
"cache_write_tokens": 0,
"cache_write_5m_tokens": 0,
"cache_write_1h_tokens": 0,
"cache_write_5m_requests": 0,
"cache_write_1h_requests": 0,
}
)
tracker = CostTracker()
tracker._tokens_sent_by_model.update({"gpt-4o": 1})
monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 100.0)
stats = build_prefix_cache_stats(metrics, tracker)
openai = stats["by_provider"]["openai"]
assert openai["savings_usd"] == 0.001
assert openai["write_premium_usd"] == 0.0
assert openai["net_savings_usd"] == openai["savings_usd"]
def test_prometheus_metrics_export_includes_extended_fields(tmp_path) -> None:
metrics = PrometheusMetrics(
savings_tracker=SavingsTracker(path=str(tmp_path / "proxy_savings.json"))