akadmin/headroom
1
0
Fork 0
mirror of https://github.com/headroomlabs-ai/headroom.git synced 2026-08-27 14:17:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 512bda982d into 8884d87378

Browse source
This commit is contained in:
Nicole Neary 2026-08-27 04:59:06 -04:00 • committed by GitHub
commit bf511baee3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 61 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
docs/content/public/pr-assets/pr-2245-cache-savings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 155 KiB

38
headroom/dashboard/templates/dashboard.html
View file

@ -289,6 +289,7 @@
x-text="formatNumber(row.events || 0) + ' calls · ' + (row.realized ? 'realized' : 'projected')"></div>
</div>
</template>
</div>
<!-- Performance -->
@ -1669,15 +1670,16 @@
</div>
</template>
<template x-if="historyModelBreakdown.length > 0">
<div class="overflow-x-auto">
<table class="w-full text-sm">
<div class="w-full overflow-hidden">
<table class="w-full table-fixed text-xs">
<thead>
<tr class="text-xs text-gray-500 text-left">
<th class="py-2 pr-4 font-medium">Model</th>
<th class="py-2 pr-4 font-medium text-right">Tokens saved</th>
<th class="py-2 pr-4 font-medium text-right">Cost with Headroom</th>
<th class="py-2 pr-4 font-medium text-right">Expected cost without Headroom</th>
<th class="py-2 font-medium text-right">Saved</th>
<tr class="text-xs text-gray-500 text-left align-bottom">
<th class="w-1/4 py-2 pr-2 font-medium">Model</th>
<th class="py-2 px-1 font-medium text-right leading-tight">Tokens saved</th>
<th class="py-2 px-1 font-medium text-right leading-tight">Cost with Headroom</th>
<th class="py-2 px-1 font-medium text-right leading-tight">Compression saved</th>
<th class="py-2 px-1 font-medium text-right leading-tight">Cache saved</th>
<th class="py-2 pl-1 font-medium text-right leading-tight">Total saved</th>
</tr>
</thead>
<tbody>
@ -1688,13 +1690,14 @@
:title="historySelectedModel === row.model
? 'Show all models'
: 'Show only this model in the chart'">
<td class="py-2 pr-4 font-mono text-xs"
<td class="py-2 pr-2 font-mono text-xs break-all"
:class="historySelectedModel === row.model ? 'underline' : ''"
x-text="truncateModel(row.model)"></td>
<td class="py-2 pr-4 text-right tabular-nums" x-text="formatNumber(row.tokens_saved)"></td>
<td class="py-2 pr-4 text-right tabular-nums" x-text="'$' + formatCurrency(row.input_cost_usd)"></td>
<td class="py-2 pr-4 text-right tabular-nums" x-text="'$' + formatCurrency(row.expected_cost_usd)"></td>
<td class="py-2 text-right tabular-nums text-accent" x-text="'$' + formatCurrency(row.savings_usd)"></td>
<td class="py-2 px-1 text-right tabular-nums" x-text="formatNumber(row.tokens_saved)"></td>
<td class="py-2 px-1 text-right tabular-nums" x-text="'$' + formatCurrency(row.input_cost_usd)"></td>
<td class="py-2 px-1 text-right tabular-nums text-emerald-300" x-text="'$' + formatCurrency(row.compression_savings_usd)"></td>
<td class="py-2 px-1 text-right tabular-nums text-cyan-400" x-text="'$' + formatCurrency(row.cache_savings_usd)"></td>
<td class="py-2 pl-1 text-right tabular-nums text-accent" x-text="'$' + formatCurrency(row.total_savings_usd)"></td>
</tr>
</template>
</tbody>
@ -2279,18 +2282,21 @@
const row = totals[model] || (totals[model] = {
model,
tokens_saved: 0,
savings_usd: 0,
compression_savings_usd: 0,
cache_savings_usd: 0,
input_cost_usd: 0,
});
row.tokens_saved += entry.tokens_saved || 0;
row.savings_usd += entry.compression_savings_usd_delta || 0;
row.compression_savings_usd += entry.compression_savings_usd_delta || 0;
row.cache_savings_usd += entry.cache_savings_usd_delta || 0;
row.input_cost_usd += entry.total_input_cost_usd_delta || 0;
}
}
return Object.values(totals)
.map(row => ({
...row,
expected_cost_usd: row.input_cost_usd + row.savings_usd,
total_savings_usd:
row.compression_savings_usd + row.cache_savings_usd,
}))
.sort((a, b) => b.tokens_saved - a.tokens_saved);
},

27
headroom/proxy/savings_tracker.py
View file

@ -1671,6 +1671,7 @@ class SavingsTracker:
aggregated: dict[str, dict[str, Any]] = {}
prev_total_tokens = 0
prev_total_usd = 0.0
prev_cache_savings_usd = 0.0
prev_total_input_tokens = 0
prev_total_input_cost_usd = 0.0
prev_output_tokens = 0
@ -1686,12 +1687,14 @@ class SavingsTracker:
bucket_key = _to_utc_iso(bucket_start)
total_tokens_saved = _coerce_int(point.get("total_tokens_saved"))
total_usd = _coerce_float(point.get("compression_savings_usd"))
cache_savings_usd = _coerce_float(point.get("cache_savings_usd"))
total_input_tokens = _coerce_int(point.get("total_input_tokens"))
total_input_cost_usd = _coerce_float(point.get("total_input_cost_usd"))
total_output_tokens = _coerce_int(point.get("output_tokens_saved"))
total_output_usd = _coerce_float(point.get("output_savings_usd"))
delta_tokens = max(total_tokens_saved - prev_total_tokens, 0)
delta_usd = max(total_usd - prev_total_usd, 0.0)
delta_cache_savings_usd = max(cache_savings_usd - prev_cache_savings_usd, 0.0)
delta_input_tokens = max(total_input_tokens - prev_total_input_tokens, 0)
delta_input_cost_usd = max(
total_input_cost_usd - prev_total_input_cost_usd,
@ -1703,6 +1706,7 @@ class SavingsTracker:
prev_total_tokens = total_tokens_saved
prev_total_usd = total_usd
prev_cache_savings_usd = cache_savings_usd
prev_total_input_tokens = total_input_tokens
prev_total_input_cost_usd = total_input_cost_usd
prev_output_tokens = total_output_tokens
@ -1714,6 +1718,7 @@ class SavingsTracker:
"timestamp": bucket_key,
"tokens_saved": 0,
"compression_savings_usd_delta": 0.0,
"cache_savings_usd_delta": 0.0,
"total_tokens_saved": total_tokens_saved,
"compression_savings_usd": total_usd,
"total_input_tokens_delta": 0,
@ -1731,6 +1736,10 @@ class SavingsTracker:
entry["compression_savings_usd_delta"] + delta_usd,
6,
)
entry["cache_savings_usd_delta"] = round(
entry["cache_savings_usd_delta"] + delta_cache_savings_usd,
6,
)
entry["total_input_tokens_delta"] += delta_input_tokens
entry["total_input_cost_usd_delta"] = round(
entry["total_input_cost_usd_delta"] + delta_input_cost_usd,
@ -1750,13 +1759,20 @@ class SavingsTracker:
# it. Each checkpoint comes from a single request, so its delta is
# wholly owned by one provider. Skip no-op checkpoints so providers
# only appear in a bucket where they actually moved a counter.
if delta_tokens or delta_usd or delta_input_tokens or delta_input_cost_usd:
if (
delta_tokens
or delta_usd
or delta_cache_savings_usd
or delta_input_tokens
or delta_input_cost_usd
):
provider = _normalize_provider(point.get("provider"))
prov = entry["by_provider"].setdefault(
provider,
{
"tokens_saved": 0,
"compression_savings_usd_delta": 0.0,
"cache_savings_usd_delta": 0.0,
"total_input_tokens_delta": 0,
"total_input_cost_usd_delta": 0.0,
},
@ -1766,6 +1782,10 @@ class SavingsTracker:
prov["compression_savings_usd_delta"] + delta_usd,
6,
)
prov["cache_savings_usd_delta"] = round(
prov["cache_savings_usd_delta"] + delta_cache_savings_usd,
6,
)
prov["total_input_tokens_delta"] += delta_input_tokens
prov["total_input_cost_usd_delta"] = round(
prov["total_input_cost_usd_delta"] + delta_input_cost_usd,
@ -1778,6 +1798,7 @@ class SavingsTracker:
{
"tokens_saved": 0,
"compression_savings_usd_delta": 0.0,
"cache_savings_usd_delta": 0.0,
"total_input_tokens_delta": 0,
"total_input_cost_usd_delta": 0.0,
},
@ -1787,6 +1808,10 @@ class SavingsTracker:
mod["compression_savings_usd_delta"] + delta_usd,
6,
)
mod["cache_savings_usd_delta"] = round(
mod["cache_savings_usd_delta"] + delta_cache_savings_usd,
6,
)
mod["total_input_tokens_delta"] += delta_input_tokens
mod["total_input_cost_usd_delta"] = round(
mod["total_input_cost_usd_delta"] + delta_input_cost_usd,

13
tests/test_proxy_savings_history.py
View file

@ -1342,6 +1342,16 @@ def test_dashboard_includes_history_toggle_and_endpoint(tmp_path, monkeypatch):
assert "Weekly Savings" in html
assert "Monthly Savings" in html
assert "Per-Model Breakdown" in html
breakdown_html = html.split("Per-Model Breakdown", 1)[1].split("Historical Summary", 1)[0]
assert "Compression saved" in breakdown_html
assert "Cache saved" in breakdown_html
assert "Total saved" in breakdown_html
assert "row.cache_savings_usd" in breakdown_html
assert "row.total_savings_usd" in breakdown_html
assert "overflow-x-auto" not in breakdown_html
assert 'class="w-full table-fixed text-xs"' in breakdown_html
assert "row.cache_savings_usd += entry.cache_savings_usd_delta || 0;" in html
assert "row.compression_savings_usd + row.cache_savings_usd" in html
assert "historyChartModeOptions" in html
assert "Expected cost (without Headroom)" in html
assert "toggleHistoryModel" in html
@ -1472,6 +1482,9 @@ def test_cache_read_savings_accumulate_and_survive_restart(tmp_path, monkeypatch
assert reloaded.snapshot()["lifetime"]["cache_savings_usd"] == pytest.approx(1.6)
assert reloaded.stats_preview()["lifetime"]["cache_read_tokens"] == 1_600_000
assert reloaded.history_response()["lifetime"]["cache_read_tokens"] == 1_600_000
daily = reloaded.history_response()["series"]["daily"]
assert daily[0]["cache_savings_usd_delta"] == pytest.approx(1.6)
assert daily[0]["by_model"]["claude-opus-4-8"]["cache_savings_usd_delta"] == pytest.approx(1.6)
def test_by_model_savings_accumulate_and_survive_restart(tmp_path):