diff --git a/CHANGELOG.md b/CHANGELOG.md index c7acbe25b..abcc697c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Fixed +- **backends/litellm:** drop tool names over 64 chars before calling Bedrock Converse (`send_message` and `stream_message`), instead of letting the whole request 401. The Bedrock Converse API hard-rejects any tool name past that length, and Claude Code includes every globally-added claude.ai MCP connector tool in every request, even ones the user hasn't enabled locally, so a single oversized connector name broke every call through this backend. Only the `bedrock` provider filters; other providers forward tool names unfiltered. - **memory:** annotate `_EMBEDDER_CACHE` as `dict[tuple[str, str, str], Embedder]` to match the 3-element key (backend, model, ollama_base_url). The stale 2-tuple annotation made `mypy headroom` fail on `main`, which broke the `lint` CI job on every open PR. - **install:** include `orjson` in the `[proxy]` extra so `uv tool install "headroom-ai[all]"` satisfies LiteLLM OpenRouter/provider backends that import it at runtime ([#2056](https://github.com/headroomlabs-ai/headroom/issues/2056)). - The dashboard's per-request metadata (the `recent_requests` / `request_logs` diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py index 1d597b544..134ae7e17 100644 --- a/headroom/backends/litellm.py +++ b/headroom/backends/litellm.py @@ -790,7 +790,14 @@ class LiteLLMBackend(Backend): # Tools (convert Anthropic format to OpenAI format) if "tools" in body: - kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]] + tools_in = body["tools"] + # Bedrock Converse API hard-rejects tool names over 64 chars. + # Claude Code injects every globally-added claude.ai MCP connector + # tool into every request, even disabled ones; a single oversized + # name 401s the whole call. Drop them before conversion instead. + if self.provider == "bedrock": + tools_in = [t for t in tools_in if len(t.get("name", "")) <= 64] + kwargs["tools"] = [_convert_anthropic_tool(t) for t in tools_in] if "tool_choice" in body: kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"]) @@ -900,7 +907,12 @@ class LiteLLMBackend(Backend): if "stop_sequences" in body: kwargs["stop"] = body["stop_sequences"] if "tools" in body: - kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]] + tools_in = body["tools"] + # Bedrock Converse API hard-rejects tool names over 64 chars. + # See send_message for the full rationale; same filter here. + if self.provider == "bedrock": + tools_in = [t for t in tools_in if len(t.get("name", "")) <= 64] + kwargs["tools"] = [_convert_anthropic_tool(t) for t in tools_in] if "tool_choice" in body: kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"]) if "system" in body: diff --git a/tests/test_backend_bugs.py b/tests/test_backend_bugs.py index bccbd25bc..ea1af8db1 100644 --- a/tests/test_backend_bugs.py +++ b/tests/test_backend_bugs.py @@ -820,3 +820,157 @@ class TestBedrockApiKeyNotForwarded: kwargs["api_key"] = headers["x-api-key"] assert "api_key" not in kwargs + + +# ============================================================================= +# Bedrock Converse Oversized Tool Name Filtering +# ============================================================================= + + +class TestBedrockOversizedToolNameFiltering: + """Bedrock Converse hard-rejects any request containing a tool name over + 64 chars. Claude Code includes every globally-added claude.ai MCP + connector tool in every request, even disabled ones, so a single + oversized name would 401 the whole call. Tools over the limit must be + dropped before the LiteLLM call, only for the ``bedrock`` provider. + """ + + def _make_response(self): + mock_response = MagicMock() + mock_response.choices = [ + MagicMock(message=MagicMock(content="ok", tool_calls=None), finish_reason="stop") + ] + mock_response.usage = MagicMock(prompt_tokens=10, completion_tokens=5) + return mock_response + + @pytest.mark.asyncio + async def test_send_message_drops_oversized_tool_name_on_bedrock(self): + with ( + patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp, + patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}), + ): + mock_acomp.return_value = self._make_response() + + backend = LiteLLMBackend(provider="bedrock", region="us-west-2") + body = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"name": "short_tool", "input_schema": {"type": "object"}}, + {"name": "x" * 65, "input_schema": {"type": "object"}}, + ], + } + + await backend.send_message(body, {}) + + call_kwargs = mock_acomp.call_args[1] + names = [t["function"]["name"] for t in call_kwargs["tools"]] + assert names == ["short_tool"] + + @pytest.mark.asyncio + async def test_send_message_keeps_exactly_64_chars_on_bedrock(self): + with ( + patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp, + patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}), + ): + mock_acomp.return_value = self._make_response() + + backend = LiteLLMBackend(provider="bedrock", region="us-west-2") + name_64 = "y" * 64 + body = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": name_64, "input_schema": {"type": "object"}}], + } + + await backend.send_message(body, {}) + + call_kwargs = mock_acomp.call_args[1] + names = [t["function"]["name"] for t in call_kwargs["tools"]] + assert names == [name_64] + + @pytest.mark.asyncio + async def test_send_message_does_not_filter_on_non_bedrock(self): + """The 64-char limit is a Bedrock Converse API constraint; other + providers must forward oversized tool names unfiltered.""" + with ( + patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp, + patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}), + ): + mock_acomp.return_value = self._make_response() + + backend = LiteLLMBackend(provider="openrouter") + oversized = "z" * 65 + body = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": oversized, "input_schema": {"type": "object"}}], + } + + await backend.send_message(body, {}) + + call_kwargs = mock_acomp.call_args[1] + names = [t["function"]["name"] for t in call_kwargs["tools"]] + assert names == [oversized] + + @pytest.mark.asyncio + async def test_stream_message_drops_oversized_tool_name_on_bedrock(self): + async def mock_stream(): + chunk = MagicMock() + chunk.choices = [ + MagicMock(delta=MagicMock(content="hi", tool_calls=None), finish_reason="stop") + ] + yield chunk + + with ( + patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp, + patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}), + ): + mock_acomp.return_value = mock_stream() + + backend = LiteLLMBackend(provider="bedrock", region="us-west-2") + body = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"name": "short_tool", "input_schema": {"type": "object"}}, + {"name": "w" * 65, "input_schema": {"type": "object"}}, + ], + } + + events = [event async for event in backend.stream_message(body, {})] + assert events # sanity: stream produced output + + call_kwargs = mock_acomp.call_args[1] + names = [t["function"]["name"] for t in call_kwargs["tools"]] + assert names == ["short_tool"] + + @pytest.mark.asyncio + async def test_stream_message_does_not_filter_on_non_bedrock(self): + async def mock_stream(): + chunk = MagicMock() + chunk.choices = [ + MagicMock(delta=MagicMock(content="hi", tool_calls=None), finish_reason="stop") + ] + yield chunk + + with ( + patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp, + patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}), + ): + mock_acomp.return_value = mock_stream() + + backend = LiteLLMBackend(provider="openrouter") + oversized = "v" * 65 + body = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": oversized, "input_schema": {"type": "object"}}], + } + + events = [event async for event in backend.stream_message(body, {})] + assert events + + call_kwargs = mock_acomp.call_args[1] + names = [t["function"]["name"] for t in call_kwargs["tools"]] + assert names == [oversized]