From 314524264512c9bc489ac17d5f45034af4fd7675 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 13 Aug 2026 17:13:23 -0700 Subject: [PATCH] Unify savings attribution across stats, perf, metrics, and dashboard (#2976) ## Summary Adds a small provider-neutral savings attribution seam. Named sources can attach realized or projected token/USD deltas to a request without changing headline arithmetic or introducing private-package inventory into OSS. Also fixes the Anthropic buffered lifecycle so normal successful responses run response hooks, applies stream-safety filtering, includes tool savings in per-model perf totals, and surfaces the same breakdown in request logs, `/stats`, `headroom perf`, Prometheus, OTEL, and the dashboard. ## Why Request-local savings were split between canonical token deltas, process-global extension counters, and tool-only tags. This made correct headline totals possible while losing attribution in perf, recent requests, metrics, and the dashboard. Normal Anthropic responses also skipped response hooks unless CCR ran. ## Validation - 74 focused tests passed: turn hooks, OpenAI hook lifecycle, outcome funnel, perf formats, and tool-search repair - Ruff passes on all changed Python files - Existing compression-observability suite: 11 passed; 2 tokenizer-cache tests require network access to fetch the tiktoken vocabulary ## Compatibility No named private packages or private inventory are encoded in OSS. Existing hooks remain source-compatible because all new TurnContext fields are optional. --------- Co-authored-by: Tejas Chopra --- headroom/cli/perf.py | 2 + headroom/dashboard/templates/dashboard.html | 34 +++- headroom/observability/__init__.py | 4 + headroom/observability/metrics.py | 141 +++++++++++++- headroom/perf/analyzer.py | 53 +++++- headroom/proxy/handlers/anthropic.py | 199 +++++++++++++++++--- headroom/proxy/handlers/openai.py | 51 ++++- headroom/proxy/models.py | 6 +- headroom/proxy/outcome.py | 10 +- headroom/proxy/prometheus_metrics.py | 88 ++++++++- headroom/proxy/savings_attribution.py | 110 +++++++++++ headroom/proxy/savings_tracker.py | 52 ++++- headroom/proxy/server.py | 7 + headroom/proxy/turn_hooks.py | 55 +++++- tests/test_observability_metrics.py | 154 +++++++++++++++ tests/test_proxy_openai.py | 2 +- tests/test_proxy_stats_recent_requests.py | 16 ++ tests/test_turn_hook_usage.py | 89 +++++++++ tests/test_turn_hooks.py | 35 ++++ 19 files changed, 1060 insertions(+), 48 deletions(-) create mode 100644 headroom/proxy/savings_attribution.py diff --git a/headroom/cli/perf.py b/headroom/cli/perf.py index 5c512a073..f3bc65112 100644 --- a/headroom/cli/perf.py +++ b/headroom/cli/perf.py @@ -79,6 +79,8 @@ def perf(hours: float, raw: bool, output_format: str) -> None: "tokens_before", "tokens_after", "tokens_saved", + "message_tokens_saved", + "tool_tokens_saved", "savings_pct", "list_price_per_mtok", ] diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.html index ca0c9efc3..4fa809431 100644 --- a/headroom/dashboard/templates/dashboard.html +++ b/headroom/dashboard/templates/dashboard.html @@ -270,20 +270,23 @@ - - +