-
Tool-Schema Deferral
+
Tokens Saved · Tool Schemas
tokens
diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.py
index 59fe4c0dd..4deec44ac 100644
--- a/headroom/perf/analyzer.py
+++ b/headroom/perf/analyzer.py
@@ -483,16 +483,25 @@ def format_report(report: PerfReport) -> str:
total_after = sum(r.tokens_after for r in records)
total_saved = sum(r.tokens_saved for r in records)
total_tool_saved = sum(r.tool_saved for r in records)
+ total_headline_saved = total_saved + total_tool_saved
pct = (total_saved / total_before * 100) if total_before > 0 else 0
+ # All-layers denominator: deferred tool schemas were never in tok_before (they
+ # don't reach count_messages), so the pre-Headroom world is tok_before + them.
+ # Same construction the proxy's /api/stats uses for total_before_compression —
+ # the headline number and the headline percent must share a numerator, or the
+ # tile reads "60,920 saved (0.1%)" off two different definitions of saved.
+ headline_before = total_before + total_tool_saved
+ headline_pct = (total_headline_saved / headline_before * 100) if headline_before > 0 else 0
lines.append(f"Requests: {len(records)}")
- lines.append(f"Tokens: {total_before:,} -> {total_after:,} ({pct:.1f}% reduction)")
- lines.append(f"Total saved: {total_saved:,} tokens (messages)")
- # Tool-schema savings (deferral + turn-hook tool shrink) are counted apart
- # from message compression — messages never include tool bytes — so surface
- # them explicitly instead of hiding a tool-heavy turn's win behind tok_saved=0.
+ lines.append(f"Tokens: {total_before:,} -> {total_after:,} ({pct:.1f}% messages)")
+ # ONE headline. Tool-schema deferral can't move tok_before/after (messages never
+ # include tool bytes), so it used to render as a rival "Tool saved" line — which
+ # read as a side metric and hid the win on tool-heavy turns where tok_saved=0.
+ lines.append(f"Tokens saved: {total_headline_saved:,} ({headline_pct:.1f}% reduction)")
if total_tool_saved > 0:
- lines.append(f"Tool saved: {total_tool_saved:,} tokens (tool schemas, deferral)")
+ lines.append(f" · messages {max(0, total_saved):,}")
+ lines.append(f" · tool schemas {total_tool_saved:,}")
lines.append("")
# Per-model breakdown with list prices
@@ -978,6 +987,7 @@ def build_perf_summary(report: PerfReport) -> dict:
total_after = sum(r.tokens_after for r in records)
total_saved = sum(r.tokens_saved for r in records)
total_tool_saved = sum(r.tool_saved for r in records)
+ total_headline_saved = total_saved + total_tool_saved
total_cr = sum(r.cache_read for r in records)
total_cw = sum(r.cache_write for r in records)
@@ -1033,6 +1043,11 @@ def build_perf_summary(report: PerfReport) -> dict:
"total_requests": len(records),
"total_tokens_before": total_before,
"total_tokens_after": total_after,
+ # total_tokens_saved is the headline (messages + tool-schema deferral); the two
+ # components stay for consumers that break the number down. See
+ # headroom.proxy.tool_schema_savings_policy.
+ "total_tokens_saved": total_headline_saved,
+ "total_savings_pct": _pct(total_headline_saved, total_before + total_tool_saved),
"tokens_saved": total_saved,
"tool_saved": total_tool_saved,
"savings_pct": _pct(total_saved, total_before),
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index cafd72edd..ee5a9092b 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -2277,14 +2277,32 @@ class AnthropicHandlerMixin:
# after tools are finalised (sorting, CCR injection) but before
# the PRE_SEND pipeline event so extensions see the compacted
# schema. Mirrors the same pass that the OpenAI handler applies.
+ #
+ # Token accounting (see tool_schema_savings_policy): the compaction passes
+ # below rewrite the tool array, so both endpoints are countable and the
+ # delta is folded into original/optimized_tokens at the final recount.
+ # Without this the savings were computed, debug-logged, and discarded —
+ # Claude Code reported them nowhere.
+ _tool_tokens_before = 0
+ _tool_tokens_after = 0
+
+ def _count_tool_tokens(value: object) -> int:
+ try:
+ return tokenizer.count_text(json.dumps(value, default=str))
+ except Exception:
+ return 0
+
_tools_compaction_started = time.time()
try:
from headroom.proxy.tool_schema_compaction import compact_tools
+ _pre_compaction_tools = body.get("tools")
body, _tools_modified, _tools_before_bytes, _tools_after_bytes = compact_tools(body)
if _tools_modified:
tools = body["tools"]
transforms_applied.append("anthropic:tool_schema_compaction")
+ _tool_tokens_before = _count_tool_tokens(_pre_compaction_tools)
+ _tool_tokens_after = _count_tool_tokens(tools)
_tools_compaction_ms = (time.time() - _tools_compaction_started) * 1000
logger.debug(
"[%s] tool schema compaction: %d -> %d bytes (%.0f%% saved) in %.1fms",
@@ -2311,12 +2329,18 @@ class AnthropicHandlerMixin:
_desc_max = tool_desc_max_chars()
if _desc_max > 0:
+ _pre_desc_tools = body.get("tools")
body, _desc_modified, _desc_before, _desc_after = compact_tool_descriptions(
body, _desc_max
)
if _desc_modified:
tools = body["tools"]
transforms_applied.append("anthropic:tool_desc_compaction")
+ # Runs after schema compaction, so only seed "before" when that
+ # pass didn't already; "after" always tracks the latest tools.
+ if not _tool_tokens_before:
+ _tool_tokens_before = _count_tool_tokens(_pre_desc_tools)
+ _tool_tokens_after = _count_tool_tokens(tools)
logger.debug(
"[%s] tool description compaction: %d -> %d bytes (%.0f%% saved, max_chars=%d)",
request_id,
@@ -2466,6 +2490,8 @@ class AnthropicHandlerMixin:
_pre_hook_tokens = tokenizer.count_messages(optimized_messages)
except Exception:
_pre_hook_tokens = None
+ _th_tools_before = body.get("tools")
+ _th_tok_before = _count_tool_tokens(_th_tools_before) if _th_tools_before else 0
run_request_hooks(_req_ctx)
if _req_ctx.messages is not optimized_messages:
optimized_messages = _req_ctx.messages
@@ -2473,6 +2499,16 @@ class AnthropicHandlerMixin:
if _req_ctx.tools is not body.get("tools"):
tools = _req_ctx.tools
body["tools"] = tools
+ # A hook may shrink the tool array either by replacing it or in place,
+ # so measure the FINAL tools object. Deferral-shaped (removes schemas
+ # count_messages never saw), hence a tag rather than a fold — mirrors
+ # the OpenAI chat path so a turn-hook extension is credited on both.
+ _th_tok_after = _count_tool_tokens(_req_ctx.tools) if _req_ctx.tools else 0
+ _th_saved = max(0, _th_tok_before - _th_tok_after)
+ if _th_saved > 0:
+ tags["turn_hook_tools_saved_tokens"] = (
+ int(tags.get("turn_hook_tools_saved_tokens", 0) or 0) + _th_saved
+ )
# Consistency: report tok_before/tok_after with ONE tokenizer. The pipeline
# and the handler use different token estimators, and cache-mode branches
@@ -2486,6 +2522,13 @@ class AnthropicHandlerMixin:
_orig_snapshot = original_client_messages # noqa: F821 (bound at request start)
original_tokens = tokenizer.count_messages(_orig_snapshot)
optimized_tokens = tokenizer.count_messages(optimized_messages)
+ # Fold the tool-schema/desc compaction delta into BOTH endpoints so
+ # tok_before - tok_after == tok_saved stays coherent in the PERF line
+ # (count_messages never sees tool bytes). Same shape as the OpenAI chat
+ # handler; guarded so a no-op or inflating pass contributes nothing.
+ if 0 < _tool_tokens_after < _tool_tokens_before:
+ original_tokens += _tool_tokens_before
+ optimized_tokens += _tool_tokens_after
tokens_saved = max(0, original_tokens - optimized_tokens)
# Attribute the fold to the hook ONLY when the hook itself reduced
# tokens (same-tokenizer pre vs post) — not when the recount above
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index 3c92b4c9b..3a4dbde96 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -30,6 +30,11 @@ from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
+from headroom.proxy.tool_schema_savings_policy import (
+ headline_tokens_saved,
+ tool_schema_saved_from_tags,
+)
+
logger = logging.getLogger("headroom.proxy")
@@ -410,10 +415,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
# Tool-schema savings (deferral + turn-hook tool shrink) live in per-request
# tags and never move tok_before/after; aggregate them into Metrics so the
# session summary / cost summary / all-layers total can surface the layer.
- _otags = outcome.tags or {}
- tool_search_saved = int(_otags.get("tool_search_deferred_tokens", 0) or 0) + int(
- _otags.get("turn_hook_tools_saved_tokens", 0) or 0
- )
+ tool_search_saved = tool_schema_saved_from_tags(outcome.tags or {})
# 1. Prometheus / SavingsTracker.
await handler.metrics.record_request(
@@ -498,21 +500,20 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
# line unchanged, and gives ``headroom perf --client X``
# parsers a clean key to filter on.
client_part = f" client={outcome.client}" if outcome.client else ""
- # Tool-schema savings are tracked separately from message compression: tool
- # deferral (defer_loading) and turn-hook tool shrink don't move tok_before/after
- # (those count messages only), so a tool-heavy turn shows tok_saved=0 while
- # genuinely saving thousands of tool-schema tokens. Surface it as its own field
- # so `headroom perf` / log readers see the whole picture.
- _tags = outcome.tags or {}
- tool_saved = int(_tags.get("tool_search_deferred_tokens", 0) or 0) + int(
- _tags.get("turn_hook_tools_saved_tokens", 0) or 0
- )
+ # Tool-schema DEFERRAL savings can't move tok_before/after (those count messages
+ # only), so a tool-heavy turn shows tok_saved=0 while genuinely saving thousands of
+ # tool-definition tokens. `tool_saved` carries that component and `total_saved` is
+ # the sum every user-facing surface reports — see tool_schema_savings_policy for why
+ # compaction is already inside tok_saved and must not be added twice.
+ tool_saved = tool_schema_saved_from_tags(outcome.tags or {})
+ total_saved = headline_tokens_saved(outcome.tokens_saved, outcome.tags or {})
logger.info(
f"[{outcome.request_id}] PERF "
f"model={outcome.model} msgs={outcome.num_messages} "
f"tok_before={outcome.original_tokens} tok_after={outcome.optimized_tokens} "
f"tok_saved={outcome.tokens_saved} "
f"tool_saved={tool_saved} "
+ f"total_saved={total_saved} "
f"cache_read={outcome.cache_read_tokens} cache_write={outcome.cache_write_tokens} "
f"cache_hit_pct={outcome.cache_hit_pct} "
f"opt_ms={outcome.overhead_ms:.0f} "
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index f2d47381e..a9ac18caa 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -1917,11 +1917,13 @@ class HeadroomProxy(
logger.info(f"Failed: {m.requests_failed}")
logger.info(f"Input tokens: {m.tokens_input_total:,}")
logger.info(f"Output tokens: {m.tokens_output_total:,}")
- logger.info(f"Tokens saved: {m.tokens_saved_total:,}")
+ # ONE headline: message compression + tool-schema deferral. Deferral can't move
+ # tok_before/after (tool bytes never reach count_messages), so it used to print
+ # as a separate line that read like a side metric rather than savings.
+ logger.info(f"Tokens saved: {m.tokens_saved_total + m.tool_search_saved_total:,}")
if m.tool_search_saved_total > 0:
- # Tool-schema deferral / turn-hook tool shrink — counted apart from
- # message compression (tool bytes never move tok_before/after).
- logger.info(f"Tool schemas deferred: {m.tool_search_saved_total:,}")
+ logger.info(f" messages: {m.tokens_saved_total:,}")
+ logger.info(f" tool schemas: {m.tool_search_saved_total:,}")
# Active-compression ratio: savings as a fraction of what we
# *attempted* to compress (extracted units + tool schema),
# NOT the whole request. The full-request denominator is
@@ -3887,9 +3889,18 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
# should show — it answers "are we doing well *when we
# have something to compress?*" rather than diluting the
# win by frozen-prefix bytes we never touched.
+ # All-layers numerator, matching denominator: `attempted_input_tokens`
+ # already counts tool schemas we COMPACTED, so pairing it with a
+ # compression-only numerator undercounted every tool-heavy session.
+ # Deferred schemas are added to both sides — they were attempted work
+ # that succeeded completely.
"active_savings_percent": round(
- (proxy_compression_tokens / attempted_input_tokens * 100)
- if attempted_input_tokens > 0
+ (
+ all_layers_tokens_saved
+ / (attempted_input_tokens + m.tool_search_saved_total)
+ * 100
+ )
+ if (attempted_input_tokens + m.tool_search_saved_total) > 0
else 0,
2,
),
diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.py
index b3ed33598..5827d0c95 100644
--- a/headroom/proxy/tool_schema_savings_policy.py
+++ b/headroom/proxy/tool_schema_savings_policy.py
@@ -1,4 +1,27 @@
-"""Tool-schema savings attribution policy for proxy stats."""
+"""Tool-schema savings attribution policy for proxy stats.
+
+Headroom saves input tokens in two accounting shapes, and the split is not a
+style choice — it follows from what each transform can observe:
+
+* **Compaction** (``*:tool_schema_compaction``, ``*:tool_desc_compaction``)
+ rewrites the tool array in place, so both endpoints are countable. Handlers
+ fold the delta into ``original_tokens``/``optimized_tokens``, which keeps
+ ``tok_before - tok_after == tok_saved`` coherent in the PERF line. It is
+ therefore ALREADY inside ``tokens_saved`` and must never be added again.
+* **Deferral / hook shrink** (tool search, turn hooks) removes schemas that
+ ``count_messages`` never saw, so it cannot move ``original_tokens``. It is
+ recorded in per-request tags and is ADDITIVE to ``tokens_saved``.
+
+The one rule a caller needs: the headline is
+``tokens_saved + tool_schema_saved_from_tags(tags)``. Use
+:func:`headline_tokens_saved` rather than open-coding it — three surfaces had
+drifted inline copies of that sum, and two harnesses were silently dropping
+their compaction savings entirely because the convention was never written down.
+
+Adding a new tool-schema-shrinking feature? If it moves the tool array, fold it
+in the handler like the compaction sites do. If it defers schemas, add its tag
+name to :data:`TOOL_SCHEMA_SAVINGS_TAGS` and every surface picks it up.
+"""
from __future__ import annotations
@@ -13,6 +36,8 @@ def tool_schema_saved_from_tags(tags: object) -> int:
The summed tags are set only on paths where Headroom performed the deferral,
so clients that already had tool search enabled contribute zero here.
+
+ These tags are additive to ``tokens_saved`` — see the module docstring.
"""
if not isinstance(tags, dict):
return 0
@@ -24,3 +49,23 @@ def tool_schema_saved_from_tags(tags: object) -> int:
except (TypeError, ValueError):
continue
return total
+
+
+def headline_tokens_saved(tokens_saved: object, tags: object) -> int:
+ """Return the single "Tokens saved" figure for one request.
+
+ This is the only correct total to show a user: message compression plus the
+ tool-definition tokens that never entered the context. Every reporting
+ surface (PERF line, ``headroom perf``, ``/api/stats``, dashboard, session
+ summary) must route through here so they cannot disagree.
+
+ Clamped at zero: handlers already revert any inflation before forwarding, so
+ a negative is a token-count artifact that never reached the model.
+ """
+ base = 0
+ if isinstance(tokens_saved, (int, float, str)) and not isinstance(tokens_saved, bool):
+ try:
+ base = int(tokens_saved)
+ except (TypeError, ValueError):
+ base = 0
+ return max(0, base + tool_schema_saved_from_tags(tags))
diff --git a/tests/test_tool_schema_savings_policy.py b/tests/test_tool_schema_savings_policy.py
index 999a7800e..a44c9b124 100644
--- a/tests/test_tool_schema_savings_policy.py
+++ b/tests/test_tool_schema_savings_policy.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from headroom.proxy.tool_schema_savings_policy import (
TOOL_SCHEMA_SAVINGS_TAGS,
+ headline_tokens_saved,
tool_schema_saved_from_tags,
)
@@ -41,3 +42,101 @@ def test_tool_schema_savings_tags_are_stable() -> None:
"tool_search_deferred_tokens",
"turn_hook_tools_saved_tokens",
)
+
+
+# ── headline_tokens_saved: the one figure every surface reports ────────────────
+# Headroom saves tool-definition tokens in two accounting shapes — compaction
+# folds into tokens_saved, deferral is tagged and additive. Both existed before
+# but the rule was never written down, so two harnesses dropped their compaction
+# savings and three surfaces open-coded the sum. These cases pin the contract.
+
+
+def test_folded_compaction_is_not_counted_twice() -> None:
+ """Compaction is ALREADY inside tokens_saved (handlers fold both endpoints).
+
+ Adding an attribution amount back on top would inflate every tool-heavy turn.
+ """
+ assert headline_tokens_saved(420, {}) == 420
+
+
+def test_deferral_tags_are_additive_to_tokens_saved() -> None:
+ """Deferral removes schemas count_messages never saw, so it can't be folded."""
+ tags = {"tool_search_deferred_tokens": 9639}
+ assert headline_tokens_saved(0, tags) == 9639
+ assert headline_tokens_saved(1_000, tags) == 10_639
+
+
+def test_headline_identical_across_harnesses_for_equivalent_work() -> None:
+ """A 500-token saving reports as 500 whichever accounting shape produced it.
+
+ Anthropic/Claude Code folds its compaction; a Codex deferral is tagged. Same
+ real saving, same headline — that equivalence is the point of the helper.
+ """
+ anthropic_folded = headline_tokens_saved(500, {})
+ codex_tagged = headline_tokens_saved(0, {"tool_search_deferred_tokens": 500})
+ assert anthropic_folded == codex_tagged == 500
+
+
+def test_headline_survives_malformed_tags() -> None:
+ for tags in (None, {}, "not-a-dict", {"tool_search_deferred_tokens": None}):
+ assert headline_tokens_saved(10, tags) == 10
+ assert headline_tokens_saved(10, {"tool_search_deferred_tokens": "abc"}) == 10
+ assert headline_tokens_saved(None, None) == 0
+
+
+def test_headline_clamps_negative_message_savings() -> None:
+ """Handlers revert inflation before forwarding, so a negative is a count artifact."""
+ assert headline_tokens_saved(-5, {}) == 0
+ assert headline_tokens_saved(-5, {"tool_search_deferred_tokens": 100}) == 95
+
+
+def test_tool_schema_compaction_saves_real_tokens_not_just_bytes() -> None:
+ """The premise of folding compaction into tokens_saved on every handler.
+
+ Compaction strips annotation keys ($schema/title/examples). If that only moved
+ bytes that tokenize to nothing, the fold would be worthless — so pin a positive
+ TOKEN delta on a realistically-shaped tool array, and pin that folding it into
+ both endpoints keeps ``tok_before - tok_after == tok_saved`` coherent.
+ """
+ import json
+
+ from headroom.providers.anthropic import AnthropicProvider
+ from headroom.proxy.tool_schema_compaction import compact_tools
+
+ tok = AnthropicProvider().get_token_counter("claude-sonnet-4-6")
+ payload = {
+ "tools": [
+ {
+ "name": f"tool_{i}",
+ "description": "Does a thing.\n\n Returns text.",
+ "input_schema": {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": f"tool_{i}_schema",
+ "examples": [{"path": "/tmp/x"}, {"path": "/tmp/y"}],
+ "type": "object",
+ "properties": {"path": {"type": "string", "description": "File path"}},
+ "required": ["path"],
+ },
+ }
+ for i in range(14)
+ ]
+ }
+ before_tools = payload["tools"]
+ body, modified, _bytes_before, _bytes_after = compact_tools(payload)
+ assert modified is True
+
+ tool_before = tok.count_text(json.dumps(before_tools, default=str))
+ tool_after = tok.count_text(json.dumps(body["tools"], default=str))
+ assert tool_after < tool_before, "compaction must shrink tool TOKENS, not only bytes"
+
+ # Mirrors the fold each handler applies at its final recount, with zero message
+ # compression — the shape that used to report tok_saved=0 on Claude Code.
+ original_tokens = optimized_tokens = 5_000
+ if 0 < tool_after < tool_before:
+ original_tokens += tool_before
+ optimized_tokens += tool_after
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+
+ assert tokens_saved == tool_before - tool_after
+ assert original_tokens - optimized_tokens == tokens_saved
+ assert headline_tokens_saved(tokens_saved, {}) == tokens_saved