From 72ade3711211183b9134a46d9c5d45db6a87edc2 Mon Sep 17 00:00:00 2001 From: gglucass Date: Tue, 30 Jun 2026 20:36:53 +0200 Subject: [PATCH] fix(savings): count cache-read tokens in input cost estimate (#1429) ## Description `_estimate_input_cost_usd` priced fully prefix-cached requests at $0. Anthropic reports cache reads/writes separately from `input_tokens` (the uncached portion), so a request served entirely from the prefix cache arrives with `input_tokens == 0` and `cache_read_tokens > 0`. The function bailed on `if total_input_tokens <= 0` *before* consulting the cache breakdown, dropping the real cache-read cost. On days dominated by cache-hit traffic this yields savings rollups with compression savings recorded but zero input tokens and zero spend. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `_estimate_input_cost_usd` now gates on tokens actually sent (`input_tokens + cache_read + cache_write + uncached`) instead of `input_tokens` alone, so cache-only requests are priced from the cache breakdown the function already supports. - Added a regression test asserting a request with `input_tokens=0, cache_read_tokens=1000` is priced at the cache-read rate rather than $0. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text $ uv run --extra dev pytest tests/test_proxy_savings_history.py -q 17 passed, 3 warnings in 24.76s $ uvx ruff check headroom/proxy/savings_tracker.py tests/test_proxy_savings_history.py All checks passed! $ uvx ruff format --check headroom/proxy/savings_tracker.py tests/test_proxy_savings_history.py 2 files already formatted $ uv run --extra dev mypy headroom/proxy/savings_tracker.py Success: no issues found in 1 source file ``` ## Real Behavior Proof - Environment: macOS, Python 3.12, headroom upstream/main - Exact command / steps: added `test_input_cost_counts_cache_reads_when_uncached_input_is_zero`; ran the suite above. The new test fails on `main` (obtains 0.0) and passes with the fix (0.3). - Observed result: cache-only requests now contribute their cache-read cost to `total_input_cost_usd`; the savings/spend invariant holds. - Not tested: no live end-to-end proxy run; the change is isolated to the cost estimator and covered by the unit test. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Additional Notes - N/A documentation / CHANGELOG: behavioral cost-accounting fix with no user-facing API or doc surface. - Follow-up (not in this PR to keep it focused): `total_input_tokens` / "tokens sent" still counts only the uncached `input_tokens` and omits cache-read tokens, so the dashboard's sent-token total under-reports cache-hit traffic. The cost fix here is sufficient to resolve the zero-spend anomaly (the probe ANDs cost == 0), but counting cache reads toward sent tokens would make the displayed total honest too. Co-authored-by: Claude Opus 4.8 --- headroom/proxy/savings_tracker.py | 12 +++++++---- tests/test_proxy_savings_history.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py index 1a6fdd945..a3549db9e 100644 --- a/headroom/proxy/savings_tracker.py +++ b/headroom/proxy/savings_tracker.py @@ -217,13 +217,17 @@ def _estimate_input_cost_usd( otherwise falls back to list-price input tokens. """ total_input_tokens = _coerce_int(input_tokens) - litellm = _get_litellm_module() - if total_input_tokens <= 0 or litellm is None: - return 0.0 - cache_read = _coerce_int(cache_read_tokens) cache_write = _coerce_int(cache_write_tokens) uncached = _coerce_int(uncached_input_tokens) + litellm = _get_litellm_module() + # Gate on tokens actually sent. Providers like Anthropic report cache + # reads/writes separately from `input_tokens` (the uncached portion), so a + # fully prefix-cached request has input_tokens == 0 while cache_read > 0. + # Bailing on `input_tokens <= 0` alone dropped the real cache-read cost, + # leaving days with compression savings but zero recorded spend. + if total_input_tokens + cache_read + cache_write + uncached <= 0 or litellm is None: + return 0.0 try: resolved = _resolve_litellm_model(model) diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py index 6b582638b..6bcc56bbd 100644 --- a/tests/test_proxy_savings_history.py +++ b/tests/test_proxy_savings_history.py @@ -343,6 +343,37 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch): assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0 +def test_input_cost_counts_cache_reads_when_uncached_input_is_zero(monkeypatch): + # Anthropic reports cache reads/writes separately from `input_tokens` (the + # uncached portion). A fully prefix-cached request has input_tokens == 0 but + # cache_read_tokens > 0 -- it still cost money and must not be priced at 0, + # otherwise the day shows compression savings with zero recorded spend. + def fake_cost_per_token(*, model, prompt_tokens, completion_tokens): + if model == "anthropic/claude-sonnet-4-6": + return {"model": model} + raise RuntimeError("unknown model") + + fake_litellm = SimpleNamespace( + cost_per_token=fake_cost_per_token, + model_cost={ + "anthropic/claude-sonnet-4-6": { + "input_cost_per_token": 0.003, + "cache_read_input_token_cost": 0.0003, + "cache_creation_input_token_cost": 0.00375, + }, + }, + ) + monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True) + monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm) + + cost = savings_tracker_module._estimate_input_cost_usd( + "claude-sonnet-4-6", + 0, + cache_read_tokens=1000, + ) + assert cost == pytest.approx(0.3) + + def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests( tmp_path, monkeypatch ):