diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py index 29a11bcf3..b737cc242 100644 --- a/headroom/proxy/handlers/openai.py +++ b/headroom/proxy/handlers/openai.py @@ -5215,6 +5215,11 @@ class OpenAIHandlerMixin: # Token counting on converted messages (offloaded off the event loop — GH #1701) tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages) + # Messages-only count, preserved for the output shaper's turn/size + # classifier: the accounting pair below may widen original_tokens by + # the tools schema after compression, and shaper strata must not + # shift when it does. + message_input_tokens = original_tokens # Defaults below feed downstream telemetry and memory injection. # If optimization remains enabled, the Responses payload is compressed @@ -5436,6 +5441,10 @@ class OpenAIHandlerMixin: # gating already happened upstream (auth_mode classify, # CompressionPolicy resolve at request entry). if self.config.optimize and not _bypass: + # Pre-compression tools reference: the compression pass is + # copy-on-write on the payload, so this still points at the + # original schemas after `body` is rebound below. + _tools_before = body.get("tools") try: ( body, @@ -5464,6 +5473,17 @@ class OpenAIHandlerMixin: if _modified: body_mutation_tracker.mark_mutated("responses_compression") tokens_saved = int(_tokens_saved) + # tokens_saved includes tool schema/desc compaction, which + # is measured against the tools array — content the + # messages-only original_tokens count above never + # included. Fold the tools schema into the pair so it + # stays coherent (original - optimized == saved, savings + # rate <= 100%); without this, schema-heavy turns clamp + # optimized to 0 and record rates above 100%. + if _tools_before: + original_tokens += await asyncio.to_thread( + tokenizer.count_text, _json_debug_dumps(_tools_before) + ) optimized_tokens = max(0, original_tokens - tokens_saved) logger.info( "[%s] /v1/responses compressed %d→%d bytes " @@ -5536,7 +5556,7 @@ class OpenAIHandlerMixin: _http_conversation_key = request.headers.get("x-headroom-session-id") _shape_result = _shape_openai_responses_for_output( body, - input_tokens=original_tokens, + input_tokens=message_input_tokens, model=str(model or ""), conversation_key=( f"header:x-headroom-session-id:{_http_conversation_key}" diff --git a/tests/test_responses_savings_denominator.py b/tests/test_responses_savings_denominator.py new file mode 100644 index 000000000..c02f30579 --- /dev/null +++ b/tests/test_responses_savings_denominator.py @@ -0,0 +1,169 @@ +"""Recorded /v1/responses savings must form a coherent triple. + +``tokens_saved`` on the Responses HTTP path includes tool schema/desc +compaction, which is measured against the ``tools`` array — content the +messages-only ``original_tokens`` count never included. Deriving +``optimized_tokens = max(0, original - saved)`` from that mismatched pair +clamped to 0 on schema-heavy turns and recorded savings rates above 100% +(surfaced in the desktop feed as e.g. "4,436 → 0 (208.4%)"). The fix folds +the tools schema into the pair, so ``original - optimized == saved`` and +the rate stays <= 100%. +""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("fastapi") +pytest.importorskip("httpx") + +import httpx +from fastapi.testclient import TestClient + +from headroom.proxy.loopback_guard import require_loopback +from headroom.proxy.server import ProxyConfig, create_app + +_TOKENS_SAVED = 5_000 + + +def _make_app(): + config = ProxyConfig( + optimize=True, + cache_enabled=False, + rate_limit_enabled=False, + cost_tracking_enabled=False, + ) + app = create_app(config) + app.dependency_overrides[require_loopback] = lambda: None + return app + + +def _fake_upstream_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "resp_test", + "object": "response", + "output": [], + "usage": {"input_tokens": 12, "output_tokens": 3}, + }, + request=httpx.Request("POST", url), + ) + + +def test_schema_heavy_compression_keeps_savings_triple_coherent(): + app = _make_app() + server = app.state.proxy + + async def _fake_retry(method, url, headers, body, stream=False, **kwargs): + return _fake_upstream_response(url) + + server._retry_request = _fake_retry + + async def _fake_compress(payload, **kwargs): + # Simulate schema compaction: savings far larger than the message + # tokens (the "4,436 -> 0" shape), sourced mostly from the tools + # array. + compressed = dict(payload) + compressed["tools"] = [{"type": "function", "name": "t", "parameters": {}}] + return ( + compressed, + True, + _TOKENS_SAVED, + ["openai:responses:tool_schema_compaction"], + None, + 100_000, + 10_000, + _TOKENS_SAVED, + {}, + ) + + server._compress_openai_responses_payload_in_executor = _fake_compress + + outcomes = [] + + async def _capture_outcome(outcome): + outcomes.append(outcome) + + server._record_request_outcome = _capture_outcome + + payload = { + "model": "gpt-5-codex", + "input": "list files", + "tools": [ + { + "type": "function", + "name": "big_tool", + # ~9k tokens of schema so original (messages + tools) can + # absorb _TOKENS_SAVED without clamping. + "description": "word " * 9_000, + "parameters": {"type": "object", "properties": {}}, + } + ], + } + + with TestClient(app) as client: + resp = client.post( + "/v1/responses", + headers={ + "Authorization": "Bearer sk-test", + "Content-Type": "application/json", + }, + json=payload, + ) + assert resp.status_code == 200 + + assert outcomes, "no RequestOutcome recorded" + o = outcomes[0] + assert o.tokens_saved == _TOKENS_SAVED + # Coherent triple: in - out == saved, out not clamped to zero, and the + # savings rate (saved / original) never above 100%. + assert o.original_tokens - o.optimized_tokens == o.tokens_saved + assert o.optimized_tokens > 0 + assert o.tokens_saved <= o.original_tokens + + +def test_no_tools_payload_keeps_messages_only_pair(): + app = _make_app() + server = app.state.proxy + + async def _fake_retry(method, url, headers, body, stream=False, **kwargs): + return _fake_upstream_response(url) + + server._retry_request = _fake_retry + + saved = 5 + + async def _fake_compress(payload, **kwargs): + return (dict(payload), True, saved, ["openai:responses:trim"], None, 1_000, 900, saved, {}) + + server._compress_openai_responses_payload_in_executor = _fake_compress + + outcomes = [] + + async def _capture_outcome(outcome): + outcomes.append(outcome) + + server._record_request_outcome = _capture_outcome + + payload = {"model": "gpt-5-codex", "input": "word " * 100} + + with TestClient(app) as client: + resp = client.post( + "/v1/responses", + headers={ + "Authorization": "Bearer sk-test", + "Content-Type": "application/json", + }, + json=payload, + ) + assert resp.status_code == 200 + + assert outcomes, "no RequestOutcome recorded" + o = outcomes[0] + assert o.tokens_saved == saved + # No tools array: original_tokens must stay the messages-only count + # (not widened), and the triple still holds. + assert o.original_tokens - o.optimized_tokens == o.tokens_saved + assert o.optimized_tokens > 0 + assert o.tokens_saved <= o.original_tokens