diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py index 455132142..39926a430 100644 --- a/headroom/proxy/handlers/openai.py +++ b/headroom/proxy/handlers/openai.py @@ -3665,8 +3665,15 @@ class OpenAIHandlerMixin: # cache stats from the LAST upstream call. total_latency = (time.time() - start_time) * 1000 usage = backend_response.body.get("usage", {}) - output_tokens = usage.get("completion_tokens", 0) - total_input_tokens = usage.get("prompt_tokens", optimized_tokens) + # `.get(key, default)` only falls back when the key is + # absent; a present-but-null count (some OpenAI-compatible + # backends emit these on a stopped/empty turn) would return + # None and crash the downstream `max(...)` arithmetic and the + # int-typed outcome/metrics. `_usage_int` coerces both cases, + # matching the streaming path and the guarded cache keys below + # (same class as the gemini fix in #2347). + output_tokens = _usage_int(usage.get("completion_tokens")) + total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens # Cache stats: prefer the Anthropic/Bedrock top-level # keys when present (authoritative). Fall back to @@ -3976,12 +3983,17 @@ class OpenAIHandlerMixin: try: resp_json = response.json() usage = resp_json.get("usage", {}) - total_input_tokens = usage.get("prompt_tokens", optimized_tokens) - output_tokens = usage.get("completion_tokens", 0) + # Coerce present-but-null counts: the arithmetic below + # (`_infer_openai_cache_write_tokens`, `max(...)`) runs + # outside this try, so a null `prompt_tokens`/`cached_tokens` + # would otherwise raise an uncaught TypeError and 500 the + # request (same class as the gemini fix in #2347). + total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens + output_tokens = _usage_int(usage.get("completion_tokens")) # OpenAI returns cached_tokens in prompt_tokens_details # These are charged at 50% of the input price prompt_details = usage.get("prompt_tokens_details") or {} - cache_read_tokens = prompt_details.get("cached_tokens", 0) + cache_read_tokens = _usage_int(prompt_details.get("cached_tokens")) except (KeyError, TypeError, AttributeError) as e: logger.debug( f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}" diff --git a/tests/test_proxy/test_openai_chat_savings_profile.py b/tests/test_proxy/test_openai_chat_savings_profile.py index 3cf8ddb2a..827d2d569 100644 --- a/tests/test_proxy/test_openai_chat_savings_profile.py +++ b/tests/test_proxy/test_openai_chat_savings_profile.py @@ -50,6 +50,67 @@ def _make_mock_backend() -> MagicMock: return backend +def _make_mock_backend_with_usage(usage: dict) -> MagicMock: + backend = MagicMock() + backend.name = "anyllm-openai" + backend.send_openai_message = AsyncMock( + return_value=BackendResponse( + body={ + "id": "chatcmpl-1", + "object": "chat.completion", + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": usage, + }, + status_code=200, + headers={"content-type": "application/json"}, + ) + ) + return backend + + +def test_chat_completions_survives_null_usage_token_counts(): + """A backend that reports present-but-null token counts must not 500. + + `.get(key, default)` returns None for a null value, and the chat path + feeds those counts into `max(...)`/int-typed metrics. Without coercion a + single such response crashes the request and its outcome recording + (same class as the gemini fix in #2347). + """ + config = ProxyConfig( + optimize=True, + cache_enabled=False, + rate_limit_enabled=False, + backend="anyllm", + anyllm_provider="openai", + ) + + # prompt_tokens / completion_tokens present but explicitly null. + mock_backend = _make_mock_backend_with_usage( + {"prompt_tokens": None, "completion_tokens": None, "total_tokens": None} + ) + with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend): + app = create_app(config) + with TestClient(app) as client: + resp = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + }, + headers={"Authorization": "Bearer test-key"}, + ) + + assert resp.status_code == 200, resp.text + + def test_chat_completions_threads_savings_profile_kwargs_into_apply(): """With HEADROOM_SAVINGS_PROFILE=agent-90, the chat path must pass the profile knobs (compress_user_messages, target_ratio, ...) to apply()."""