akadmin/headroom
1
0
Fork 0
mirror of https://github.com/headroomlabs-ai/headroom.git synced 2026-08-27 14:17:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(proxy/openai): keep /v1/responses savings triple coherent when schema savings exceed messages

Browse source 
tokens_saved on the Responses HTTP path includes tool schema/desc
compaction, measured against the tools array - content the messages-only
original_tokens count never included. Deriving optimized_tokens by
subtraction from that mismatched pair clamped to 0 on schema-heavy turns
and recorded savings rates above 100% (e.g. 4,436 -> 0, 208.4% in the
desktop feed). Fold the tools schema into the pair after compression so
original - optimized == saved and the rate stays <= 100%. The output
shaper keeps classifying on the messages-only count so strata labels do
not shift.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garm 2026-08-18 14:03:18 +02:00
parent 7ef736fb1a
commit 6bcfacc539
2 changed files with 190 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

22
headroom/proxy/handlers/openai.py
View file

@ -5215,6 +5215,11 @@ class OpenAIHandlerMixin:
# Token counting on converted messages (offloaded off the event loop — GH #1701)
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
# Messages-only count, preserved for the output shaper's turn/size
# classifier: the accounting pair below may widen original_tokens by
# the tools schema after compression, and shaper strata must not
# shift when it does.
message_input_tokens = original_tokens
# Defaults below feed downstream telemetry and memory injection.
# If optimization remains enabled, the Responses payload is compressed
@ -5436,6 +5441,10 @@ class OpenAIHandlerMixin:
# gating already happened upstream (auth_mode classify,
# CompressionPolicy resolve at request entry).
if self.config.optimize and not _bypass:
# Pre-compression tools reference: the compression pass is
# copy-on-write on the payload, so this still points at the
# original schemas after `body` is rebound below.
_tools_before = body.get("tools")
try:
(
body,
@ -5464,6 +5473,17 @@ class OpenAIHandlerMixin:
if _modified:
body_mutation_tracker.mark_mutated("responses_compression")
tokens_saved = int(_tokens_saved)
# tokens_saved includes tool schema/desc compaction, which
# is measured against the tools array — content the
# messages-only original_tokens count above never
# included. Fold the tools schema into the pair so it
# stays coherent (original - optimized == saved, savings
# rate <= 100%); without this, schema-heavy turns clamp
# optimized to 0 and record rates above 100%.
if _tools_before:
original_tokens += await asyncio.to_thread(
tokenizer.count_text, _json_debug_dumps(_tools_before)
)
optimized_tokens = max(0, original_tokens - tokens_saved)
logger.info(
"[%s] /v1/responses compressed %d→%d bytes "
@ -5536,7 +5556,7 @@ class OpenAIHandlerMixin:
_http_conversation_key = request.headers.get("x-headroom-session-id")
_shape_result = _shape_openai_responses_for_output(
body,
input_tokens=original_tokens,
input_tokens=message_input_tokens,
model=str(model or ""),
conversation_key=(
f"header:x-headroom-session-id:{_http_conversation_key}"

169
tests/test_responses_savings_denominator.py Normal file
View file

@ -0,0 +1,169 @@
"""Recorded /v1/responses savings must form a coherent triple.
``tokens_saved`` on the Responses HTTP path includes tool schema/desc
compaction, which is measured against the ``tools`` array — content the
messages-only ``original_tokens`` count never included. Deriving
``optimized_tokens = max(0, original - saved)`` from that mismatched pair
clamped to 0 on schema-heavy turns and recorded savings rates above 100%
(surfaced in the desktop feed as e.g. "4,436 → 0 (208.4%)"). The fix folds
the tools schema into the pair, so ``original - optimized == saved`` and
the rate stays <= 100%.
"""
from __future__ import annotations
import pytest
pytest.importorskip("fastapi")
pytest.importorskip("httpx")
import httpx
from fastapi.testclient import TestClient
from headroom.proxy.loopback_guard import require_loopback
from headroom.proxy.server import ProxyConfig, create_app
_TOKENS_SAVED = 5_000
def _make_app():
config = ProxyConfig(
optimize=True,
cache_enabled=False,
rate_limit_enabled=False,
cost_tracking_enabled=False,
)
app = create_app(config)
app.dependency_overrides[require_loopback] = lambda: None
return app
def _fake_upstream_response(url: str) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "resp_test",
"object": "response",
"output": [],
"usage": {"input_tokens": 12, "output_tokens": 3},
},
request=httpx.Request("POST", url),
)
def test_schema_heavy_compression_keeps_savings_triple_coherent():
app = _make_app()
server = app.state.proxy
async def _fake_retry(method, url, headers, body, stream=False, **kwargs):
return _fake_upstream_response(url)
server._retry_request = _fake_retry
async def _fake_compress(payload, **kwargs):
# Simulate schema compaction: savings far larger than the message
# tokens (the "4,436 -> 0" shape), sourced mostly from the tools
# array.
compressed = dict(payload)
compressed["tools"] = [{"type": "function", "name": "t", "parameters": {}}]
return (
compressed,
True,
_TOKENS_SAVED,
["openai:responses:tool_schema_compaction"],
None,
100_000,
10_000,
_TOKENS_SAVED,
{},
)
server._compress_openai_responses_payload_in_executor = _fake_compress
outcomes = []
async def _capture_outcome(outcome):
outcomes.append(outcome)
server._record_request_outcome = _capture_outcome
payload = {
"model": "gpt-5-codex",
"input": "list files",
"tools": [
{
"type": "function",
"name": "big_tool",
# ~9k tokens of schema so original (messages + tools) can
# absorb _TOKENS_SAVED without clamping.
"description": "word " * 9_000,
"parameters": {"type": "object", "properties": {}},
}
],
}
with TestClient(app) as client:
resp = client.post(
"/v1/responses",
headers={
"Authorization": "Bearer sk-test",
"Content-Type": "application/json",
},
json=payload,
)
assert resp.status_code == 200
assert outcomes, "no RequestOutcome recorded"
o = outcomes[0]
assert o.tokens_saved == _TOKENS_SAVED
# Coherent triple: in - out == saved, out not clamped to zero, and the
# savings rate (saved / original) never above 100%.
assert o.original_tokens - o.optimized_tokens == o.tokens_saved
assert o.optimized_tokens > 0
assert o.tokens_saved <= o.original_tokens
def test_no_tools_payload_keeps_messages_only_pair():
app = _make_app()
server = app.state.proxy
async def _fake_retry(method, url, headers, body, stream=False, **kwargs):
return _fake_upstream_response(url)
server._retry_request = _fake_retry
saved = 5
async def _fake_compress(payload, **kwargs):
return (dict(payload), True, saved, ["openai:responses:trim"], None, 1_000, 900, saved, {})
server._compress_openai_responses_payload_in_executor = _fake_compress
outcomes = []
async def _capture_outcome(outcome):
outcomes.append(outcome)
server._record_request_outcome = _capture_outcome
payload = {"model": "gpt-5-codex", "input": "word " * 100}
with TestClient(app) as client:
resp = client.post(
"/v1/responses",
headers={
"Authorization": "Bearer sk-test",
"Content-Type": "application/json",
},
json=payload,
)
assert resp.status_code == 200
assert outcomes, "no RequestOutcome recorded"
o = outcomes[0]
assert o.tokens_saved == saved
# No tools array: original_tokens must stay the messages-only count
# (not widened), and the triple still holds.
assert o.original_tokens - o.optimized_tokens == o.tokens_saved
assert o.optimized_tokens > 0
assert o.tokens_saved <= o.original_tokens