mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
feat(dashboard): surface compression-vs-cache net impact in Prefix Cache panel (#913)
## Summary Closes #911, and delivers the dashboard half of the question in #855 ("how do I understand the cache impact — can we add it to the dashboard?"). `GET /stats` already exposes `prefix_cache.compression_vs_cache` (`tokens_saved_by_compression`, `tokens_lost_to_cache_bust`, `cache_bust_count`, `net_tokens`) and `prefix_cache.prefix_freeze` (`busts_avoided`, `tokens_preserved`, `compression_foregone_tokens`, `net_benefit_tokens`) — built in `headroom/proxy/cost.py` — but the dashboard never rendered them. This adds a **Compression vs Cache** section to the existing Prefix Cache Impact panel: - **Saved by Compression** — tokens removed before send - **Lost to Cache Busts** — tokens lost, with observed bust count - **Net** — color-coded emerald when positive, red when negative, with a matching "Net positive / Net negative" headline pill - **Prefix Freeze Net** — net benefit of freeze decisions (preserved minus compression foregone), with busts avoided The section is gated on data presence (hidden until any underlying counter is non-zero), styled to match the existing TTL bucket cards, works in light and dark mode, and carries `data-testid` hooks. Frontend-only: no backend changes; the stats endpoint already serves every field. ## Testing New `tests/test_dashboard_cache_net_playwright.py` (mirrors the TTL playwright test): pins the rendered values, the negative-net red styling, and the hidden-when-empty behavior. 3/3 pass locally under chromium. Note for reviewers: the harness matches stubbed routes on URL **path**, because the dashboard now fetches `/stats?cached=1` — full-URL `endswith("/stats")` checks miss it and the request escapes to the real network. The pre-existing `test_dashboard_cache_ttl_playwright.py` has this exact bitrot (plus stale text assertions) and currently fails when playwright is actually installed — playwright isn't installed in CI so it silently skips. Left out of scope here; can file separately. `ruff check` and `ruff format --check` clean. Refs #855. Co-authored-by: integration-check <integration@local>
This commit is contained in:
parent
4f59097045
commit
2a4d300841
2 changed files with 207 additions and 0 deletions
|
|
@ -778,6 +778,40 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
<!-- Compression vs cache: net impact of prefix-mutating layers -->
|
||||||
|
<template x-if="hasCompressionVsCache">
|
||||||
|
<div class="mt-5 border-t border-border pt-4">
|
||||||
|
<div class="flex items-center justify-between mb-3">
|
||||||
|
<div>
|
||||||
|
<div class="text-xs text-gray-500 uppercase tracking-wide">Compression vs Cache</div>
|
||||||
|
<div class="text-[11px] text-gray-600 mt-0.5">Tokens saved by compression against cached-prefix tokens its mutations invalidated</div>
|
||||||
|
</div>
|
||||||
|
<div data-testid="cvc-net-headline" class="rounded-full border bg-black/20 px-3 py-1 text-[11px] uppercase tracking-[0.18em]" :class="compressionVsCacheNet >= 0 ? 'border-emerald-400/20 text-emerald-300/90' : 'border-red-400/25 text-red-300/90'" x-text="compressionVsCacheNet >= 0 ? 'Net positive' : 'Net negative'"></div>
|
||||||
|
</div>
|
||||||
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-3">
|
||||||
|
<div class="rounded-2xl border border-white/8 bg-black/20 p-4">
|
||||||
|
<div class="text-[11px] uppercase tracking-[0.18em] text-emerald-300/75">Saved by Compression</div>
|
||||||
|
<div data-testid="cvc-saved-value" class="mt-2 text-3xl font-light text-emerald-50" x-text="formatNumber(stats.prefix_cache?.compression_vs_cache?.tokens_saved_by_compression || 0)"></div>
|
||||||
|
<div class="mt-1 text-xs text-gray-500">tokens removed before send</div>
|
||||||
|
</div>
|
||||||
|
<div class="rounded-2xl border border-white/8 bg-black/20 p-4">
|
||||||
|
<div class="text-[11px] uppercase tracking-[0.18em] text-red-300/75">Lost to Cache Busts</div>
|
||||||
|
<div data-testid="cvc-bust-value" class="mt-2 text-3xl font-light text-red-50" x-text="formatNumber(stats.prefix_cache?.compression_vs_cache?.tokens_lost_to_cache_bust || 0)"></div>
|
||||||
|
<div class="mt-1 text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.compression_vs_cache?.cache_bust_count || 0) + ' busts observed'"></div>
|
||||||
|
</div>
|
||||||
|
<div class="rounded-2xl border border-white/8 bg-black/20 p-4">
|
||||||
|
<div class="text-[11px] uppercase tracking-[0.18em] text-gray-400">Net</div>
|
||||||
|
<div data-testid="cvc-net-value" class="mt-2 text-3xl font-light" :class="compressionVsCacheNet >= 0 ? 'text-emerald-400' : 'text-red-400'" x-text="(compressionVsCacheNet >= 0 ? '+' : '-') + formatNumber(Math.abs(compressionVsCacheNet))"></div>
|
||||||
|
<div class="mt-1 text-xs text-gray-500">saved minus bust losses</div>
|
||||||
|
</div>
|
||||||
|
<div class="rounded-2xl border border-white/8 bg-black/20 p-4">
|
||||||
|
<div class="text-[11px] uppercase tracking-[0.18em] text-cyan-300/75">Prefix Freeze Net</div>
|
||||||
|
<div data-testid="freeze-net-value" class="mt-2 text-3xl font-light" :class="prefixFreezeNet >= 0 ? 'text-cyan-50' : 'text-red-400'" x-text="(prefixFreezeNet >= 0 ? '+' : '-') + formatNumber(Math.abs(prefixFreezeNet))"></div>
|
||||||
|
<div class="mt-1 text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.prefix_freeze?.busts_avoided || 0) + ' busts avoided, ' + formatNumber(stats.prefix_cache?.prefix_freeze?.compression_foregone_tokens || 0) + ' foregone'"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
<!-- Per-provider breakdown -->
|
<!-- Per-provider breakdown -->
|
||||||
<template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
|
<template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
|
||||||
<div class="mt-4 border-t border-border pt-3">
|
<div class="mt-4 border-t border-border pt-3">
|
||||||
|
|
@ -2225,6 +2259,25 @@
|
||||||
return active.length === 1 ? active[0] + ' only' : active.join(' / ');
|
return active.length === 1 ? active[0] + ' only' : active.join(' / ');
|
||||||
},
|
},
|
||||||
|
|
||||||
|
get compressionVsCacheNet() {
|
||||||
|
const cvc = this.stats.prefix_cache?.compression_vs_cache || {};
|
||||||
|
return cvc.net_tokens ?? ((cvc.tokens_saved_by_compression || 0) - (cvc.tokens_lost_to_cache_bust || 0));
|
||||||
|
},
|
||||||
|
|
||||||
|
get prefixFreezeNet() {
|
||||||
|
const pf = this.stats.prefix_cache?.prefix_freeze || {};
|
||||||
|
return pf.net_benefit_tokens ?? ((pf.tokens_preserved || 0) - (pf.compression_foregone_tokens || 0));
|
||||||
|
},
|
||||||
|
|
||||||
|
get hasCompressionVsCache() {
|
||||||
|
const cvc = this.stats.prefix_cache?.compression_vs_cache || {};
|
||||||
|
const pf = this.stats.prefix_cache?.prefix_freeze || {};
|
||||||
|
return (cvc.tokens_saved_by_compression || 0) > 0
|
||||||
|
|| (cvc.tokens_lost_to_cache_bust || 0) > 0
|
||||||
|
|| (pf.tokens_preserved || 0) > 0
|
||||||
|
|| (pf.compression_foregone_tokens || 0) > 0;
|
||||||
|
},
|
||||||
|
|
||||||
// --- Waste Signals ---
|
// --- Waste Signals ---
|
||||||
|
|
||||||
wasteSignalLabel(signal) {
|
wasteSignalLabel(signal) {
|
||||||
|
|
|
||||||
154
tests/test_dashboard_cache_net_playwright.py
Normal file
154
tests/test_dashboard_cache_net_playwright.py
Normal file
|
|
@ -0,0 +1,154 @@
|
||||||
|
"""Behavior-driven Playwright validation for the Compression vs Cache panel.
|
||||||
|
|
||||||
|
The /stats endpoint has long exposed ``prefix_cache.compression_vs_cache``
|
||||||
|
and ``prefix_cache.prefix_freeze`` (built in ``headroom/proxy/cost.py``)
|
||||||
|
but the dashboard never rendered them. These tests pin the new section:
|
||||||
|
net tokens saved by compression against cached-prefix tokens its mutations
|
||||||
|
invalidated, plus the prefix-freeze net benefit.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from headroom.dashboard import get_dashboard_html
|
||||||
|
from tests.test_dashboard_cache_ttl_playwright import _sample_history, _sample_stats
|
||||||
|
|
||||||
|
playwright = pytest.importorskip("playwright.sync_api")
|
||||||
|
Page = playwright.Page
|
||||||
|
expect = playwright.expect
|
||||||
|
sync_playwright = playwright.sync_playwright
|
||||||
|
|
||||||
|
|
||||||
|
def _stats_with_compression_vs_cache() -> dict:
|
||||||
|
stats = copy.deepcopy(_sample_stats())
|
||||||
|
stats["prefix_cache"]["compression_vs_cache"] = {
|
||||||
|
"tokens_saved_by_compression": 143_000,
|
||||||
|
"tokens_lost_to_cache_bust": 36_000,
|
||||||
|
"cache_bust_count": 4,
|
||||||
|
"net_tokens": 107_000,
|
||||||
|
}
|
||||||
|
stats["prefix_cache"]["prefix_freeze"] = {
|
||||||
|
"busts_avoided": 6,
|
||||||
|
"tokens_preserved": 88_000,
|
||||||
|
"compression_foregone_tokens": 21_000,
|
||||||
|
"net_benefit_tokens": 67_000,
|
||||||
|
}
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def _install_dashboard_routes(page: Page, stats: dict) -> None:
|
||||||
|
history = _sample_history()
|
||||||
|
health = {"status": "healthy", "version": "0.3.0"}
|
||||||
|
dashboard_html = get_dashboard_html()
|
||||||
|
|
||||||
|
def handler(route) -> None: # type: ignore[no-untyped-def]
|
||||||
|
# Match on the URL path only: the dashboard fetches /stats?cached=1,
|
||||||
|
# so suffix checks against the full URL miss it and the request
|
||||||
|
# escapes the harness to the real network.
|
||||||
|
path = urlsplit(route.request.url).path
|
||||||
|
if path in ("/dashboard", "/"):
|
||||||
|
route.fulfill(status=200, content_type="text/html", body=dashboard_html)
|
||||||
|
return
|
||||||
|
if "/stats-history" in path:
|
||||||
|
route.fulfill(status=200, content_type="application/json", body=json.dumps(history))
|
||||||
|
return
|
||||||
|
if path.endswith("/stats"):
|
||||||
|
route.fulfill(status=200, content_type="application/json", body=json.dumps(stats))
|
||||||
|
return
|
||||||
|
if path.endswith("/health"):
|
||||||
|
route.fulfill(status=200, content_type="application/json", body=json.dumps(health))
|
||||||
|
return
|
||||||
|
route.continue_()
|
||||||
|
|
||||||
|
page.route("**/*", handler)
|
||||||
|
|
||||||
|
|
||||||
|
def _open_dashboard(page: Page, stats: dict) -> None:
|
||||||
|
_install_dashboard_routes(page, stats)
|
||||||
|
page.goto("http://headroom.local/dashboard")
|
||||||
|
page.wait_for_load_state("networkidle")
|
||||||
|
|
||||||
|
|
||||||
|
def test_dashboard_renders_compression_vs_cache_net_metrics() -> None:
|
||||||
|
artifact_dir = os.environ.get("HEADROOM_PLAYWRIGHT_ARTIFACT_DIR")
|
||||||
|
|
||||||
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch()
|
||||||
|
page = browser.new_page(viewport={"width": 1440, "height": 1600})
|
||||||
|
_open_dashboard(page, _stats_with_compression_vs_cache())
|
||||||
|
|
||||||
|
expect(page.get_by_text("Compression vs Cache", exact=True)).to_be_visible()
|
||||||
|
expect(page.get_by_test_id("cvc-net-headline")).to_have_text("Net positive")
|
||||||
|
expect(page.get_by_test_id("cvc-saved-value")).to_have_text("143.0k")
|
||||||
|
expect(page.get_by_test_id("cvc-bust-value")).to_have_text("36.0k")
|
||||||
|
expect(page.get_by_text("4 busts observed")).to_be_visible()
|
||||||
|
expect(page.get_by_test_id("cvc-net-value")).to_have_text("+107.0k")
|
||||||
|
expect(page.get_by_test_id("cvc-net-value")).to_have_class(
|
||||||
|
"mt-2 text-3xl font-light text-emerald-400"
|
||||||
|
)
|
||||||
|
expect(page.get_by_test_id("freeze-net-value")).to_have_text("+67.0k")
|
||||||
|
expect(page.get_by_text("6 busts avoided, 21.0k foregone")).to_be_visible()
|
||||||
|
|
||||||
|
if artifact_dir:
|
||||||
|
Path(artifact_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
page.screenshot(
|
||||||
|
path=str(Path(artifact_dir) / "dashboard-compression-vs-cache.png"),
|
||||||
|
full_page=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
browser.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_dashboard_marks_negative_compression_vs_cache_net_in_red() -> None:
|
||||||
|
stats = _stats_with_compression_vs_cache()
|
||||||
|
stats["prefix_cache"]["compression_vs_cache"] = {
|
||||||
|
"tokens_saved_by_compression": 12_000,
|
||||||
|
"tokens_lost_to_cache_bust": 48_000,
|
||||||
|
"cache_bust_count": 9,
|
||||||
|
"net_tokens": -36_000,
|
||||||
|
}
|
||||||
|
|
||||||
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch()
|
||||||
|
page = browser.new_page(viewport={"width": 1440, "height": 1600})
|
||||||
|
_open_dashboard(page, stats)
|
||||||
|
|
||||||
|
expect(page.get_by_test_id("cvc-net-headline")).to_have_text("Net negative")
|
||||||
|
expect(page.get_by_test_id("cvc-net-value")).to_have_text("-36.0k")
|
||||||
|
expect(page.get_by_test_id("cvc-net-value")).to_have_class(
|
||||||
|
"mt-2 text-3xl font-light text-red-400"
|
||||||
|
)
|
||||||
|
|
||||||
|
browser.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_dashboard_hides_compression_vs_cache_section_without_data() -> None:
|
||||||
|
stats = copy.deepcopy(_sample_stats())
|
||||||
|
stats["prefix_cache"]["compression_vs_cache"] = {
|
||||||
|
"tokens_saved_by_compression": 0,
|
||||||
|
"tokens_lost_to_cache_bust": 0,
|
||||||
|
"cache_bust_count": 0,
|
||||||
|
"net_tokens": 0,
|
||||||
|
}
|
||||||
|
stats["prefix_cache"]["prefix_freeze"] = {
|
||||||
|
"busts_avoided": 0,
|
||||||
|
"tokens_preserved": 0,
|
||||||
|
"compression_foregone_tokens": 0,
|
||||||
|
"net_benefit_tokens": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch()
|
||||||
|
page = browser.new_page(viewport={"width": 1440, "height": 1600})
|
||||||
|
_open_dashboard(page, stats)
|
||||||
|
|
||||||
|
expect(page.get_by_test_id("cvc-net-headline")).to_have_count(0)
|
||||||
|
|
||||||
|
browser.close()
|
||||||
Loading…
Add table
Add a link
Reference in a new issue