diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.py index b5f8487f3..33cb13866 100644 --- a/headroom/perf/analyzer.py +++ b/headroom/perf/analyzer.py @@ -1038,6 +1038,7 @@ def build_perf_summary(report: PerfReport) -> dict: total_before = sum(r.tokens_before for r in records) total_after = sum(r.tokens_after for r in records) total_saved = sum(r.tokens_saved for r in records) + total_tool_saved = sum(r.tool_saved for r in records) total_cr = sum(r.cache_read for r in records) total_cw = sum(r.cache_write for r in records) @@ -1094,6 +1095,7 @@ def build_perf_summary(report: PerfReport) -> dict: "total_tokens_before": total_before, "total_tokens_after": total_after, "tokens_saved": total_saved, + "tool_saved": total_tool_saved, "savings_pct": _pct(total_saved, total_before), "cache_read_tokens": total_cr, "cache_write_tokens": total_cw, diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py index e9fcdf63c..4a0539e29 100644 --- a/headroom/proxy/cost.py +++ b/headroom/proxy/cost.py @@ -593,6 +593,15 @@ def build_session_summary( "rtk_tokens_avoided": cli_tokens_avoided, "total_tokens_saved_with_rtk": metrics.tokens_saved_total + cli_tokens_avoided, "total_tokens_before_with_rtk": total_tokens_before, + # Tool-schema deferral / turn-hook tool shrink, tracked apart from + # message compression. New fields (existing ones stay message+CLI only + # for backward compat) so consumers can see the full picture. + "tool_schema_tokens_saved": getattr(metrics, "tool_search_saved_total", 0), + "total_tokens_saved_all_layers": ( + metrics.tokens_saved_total + + cli_tokens_avoided + + getattr(metrics, "tool_search_saved_total", 0) + ), }, "uncompressed_requests": {k: v for k, v in uncompressed_reasons.items() if v > 0}, "cost": { diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py index 41a87dc4a..e789d452a 100644 --- a/headroom/proxy/outcome.py +++ b/headroom/proxy/outcome.py @@ -388,6 +388,14 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: # HTTP middleware / WS accept captured from ``X-Headroom-Project``. project = outcome.project or get_current_project() + # Tool-schema savings (deferral + turn-hook tool shrink) live in per-request + # tags and never move tok_before/after; aggregate them into Metrics so the + # session summary / cost summary / all-layers total can surface the layer. + _otags = outcome.tags or {} + tool_search_saved = int(_otags.get("tool_search_deferred_tokens", 0) or 0) + int( + _otags.get("turn_hook_tools_saved_tokens", 0) or 0 + ) + # 1. Prometheus / SavingsTracker. await handler.metrics.record_request( provider=outcome.provider, @@ -410,6 +418,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: output_tokens_saved=output_tokens_saved_est, project=project, client=outcome.client, + tool_search_saved=tool_search_saved, ) # 2. Cost tracker (optional). diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py index 4413feec5..5ca179914 100644 --- a/headroom/proxy/prometheus_metrics.py +++ b/headroom/proxy/prometheus_metrics.py @@ -97,6 +97,11 @@ class PrometheusMetrics: self.tokens_input_total = 0 self.tokens_output_total = 0 self.tokens_saved_total = 0 + # Tool-schema savings (deferral + turn-hook tool shrink), aggregated from + # per-request tags. Tracked apart from tokens_saved_total (which is message + # compression only — tool bytes never move tok_before/after) so every sink + # can surface the tool-schema layer instead of silently dropping it. + self.tool_search_saved_total = 0 # Sum of tokens we actually attempted to compress across the # session: extracted units that passed all gates + tool-schema # tokens we ran compaction against. Excludes prefix-frozen @@ -324,6 +329,7 @@ class PrometheusMetrics: self.tokens_input_total = 0 self.tokens_output_total = 0 self.tokens_saved_total = 0 + self.tool_search_saved_total = 0 self.attempted_input_tokens_total = 0 self.compressions_by_strategy.clear() @@ -683,6 +689,7 @@ class PrometheusMetrics: output_tokens_saved: int = 0, project: str | None = None, client: str | None = None, + tool_search_saved: int = 0, ): """Record metrics for a request.""" # Post-guard invariant (all providers): Headroom never forwards a request @@ -709,6 +716,7 @@ class PrometheusMetrics: self.tokens_input_total += input_tokens self.tokens_output_total += output_tokens self.tokens_saved_total += tokens_saved + self.tool_search_saved_total += max(0, int(tool_search_saved)) # See the attribute definition for why this is the right # denominator for the active-compression ratio. self.attempted_input_tokens_total += max(0, int(attempted_input_tokens)) diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index 3e4afbfa9..b069335e0 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -1897,6 +1897,10 @@ class HeadroomProxy( logger.info(f"Input tokens: {m.tokens_input_total:,}") logger.info(f"Output tokens: {m.tokens_output_total:,}") logger.info(f"Tokens saved: {m.tokens_saved_total:,}") + if m.tool_search_saved_total > 0: + # Tool-schema deferral / turn-hook tool shrink — counted apart from + # message compression (tool bytes never move tok_before/after). + logger.info(f"Tool schemas deferred: {m.tool_search_saved_total:,}") # Active-compression ratio: savings as a fraction of what we # *attempted* to compress (extracted units + tool schema), # NOT the whole request. The full-request denominator is @@ -3647,7 +3651,11 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: # compression and the configured context tool both remove tokens before # they reach model context, so dashboard-facing savings combines them. proxy_compression_tokens = m.tokens_saved_total - all_layers_tokens_saved = proxy_compression_tokens + cli_tokens_avoided + # "All layers" must include tool-schema deferral (the tool_search layer + # enumerated in by_layer below) — otherwise the advertised total omits it. + all_layers_tokens_saved = ( + proxy_compression_tokens + cli_tokens_avoided + m.tool_search_saved_total + ) total_tokens_before = m.tokens_input_total + all_layers_tokens_saved proxy_total_before_compression = m.tokens_input_total + proxy_compression_tokens # `attempted_input_tokens` is the compressible-only denominator diff --git a/tests/test_savings_tool_search_aggregation.py b/tests/test_savings_tool_search_aggregation.py new file mode 100644 index 000000000..21d4300c1 --- /dev/null +++ b/tests/test_savings_tool_search_aggregation.py @@ -0,0 +1,67 @@ +"""Tool-search / deferral savings must aggregate into Metrics and surface in the +reporting sinks — not live only in per-request tags (which every sink reading +metrics.* structurally missed: session summary, cost summary, all-layers total, +`headroom perf --json`).""" + +from __future__ import annotations + +import asyncio + +from headroom.perf.analyzer import PerfRecord, PerfReport, build_perf_summary +from headroom.proxy.prometheus_metrics import PrometheusMetrics + + +def test_metrics_accumulates_tool_search_saved_apart_from_message() -> None: + m = PrometheusMetrics() + + async def go() -> None: + await m.record_request( + provider="anthropic", + model="claude-x", + input_tokens=100, + output_tokens=10, + tokens_saved=0, + latency_ms=1.0, + tool_search_saved=1500, + ) + await m.record_request( + provider="anthropic", + model="claude-x", + input_tokens=100, + output_tokens=10, + tokens_saved=200, + latency_ms=1.0, + tool_search_saved=800, + ) + + asyncio.run(go()) + assert m.tokens_saved_total == 200 # message compression only + assert m.tool_search_saved_total == 2300 # tool-schema layer, aggregated + + +def test_build_perf_summary_includes_tool_saved() -> None: + report = PerfReport( + perf_records=[ + PerfRecord( + timestamp="t", + request_id="r1", + model="m", + tokens_before=1000, + tokens_after=900, + tokens_saved=100, + tool_saved=5000, + ), + PerfRecord( + timestamp="t", + request_id="r2", + model="m", + tokens_before=500, + tokens_after=500, + tokens_saved=0, + tool_saved=3000, + ), + ] + ) + summary = build_perf_summary(report) + assert summary["tokens_saved"] == 100 # message + assert summary["tool_saved"] == 8000 # tool-schema surfaced in json/csv sink