From ce8ce8313f8cebf060392a62f9adaab18c0df386 Mon Sep 17 00:00:00 2001 From: Parideboy Date: Wed, 12 Aug 2026 07:18:43 +0200 Subject: [PATCH] fix(ccr): resolve <> markers inline when no retrieve-tool path exists (#2512) ## Description Fixes #2509. CCR marker resolution today depends entirely on the model calling `headroom_retrieve` back a tool-call round-trip. Callers with no such round-trip (e.g. Headroom running as a LiteLLM guardrail/proxy hop, per the issue's repro) never get an offered path to redeem a marker, so raw `<>` text leaks straight to the agent. This adds an explicit, opt-in fallback: `--ccr-inline-resolve` / `HEADROOM_CCR_INLINE_RESOLVE`. When set, the proxy resolves markers directly from the compression store on the response path instead of waiting for a tool call. Off by default, guessing "this caller can't use tools" is fragile, so operators opt in explicitly for guardrail/proxy deployments. ## Type of Change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change - [ ] Documentation update ## Changes Made - `headroom/ccr/marker_resolution.py` (new): `resolve_markers_in_text` / `resolve_markers_in_response` regex-match `<>`, look up the hash in `CompressionStore`, splice the original content back in. A miss (expired/evicted hash) leaves the marker in place with the miss reason appended, since there's no tool-call round-trip to report it back to the model. - `headroom/proxy/models.py`: `ProxyConfig.ccr_resolve_markers_inline: bool = False`. - `headroom/cli/proxy.py`: `--ccr-inline-resolve` flag / `HEADROOM_CCR_INLINE_RESOLVE` env, wired into `ProxyConfig`. - `headroom/proxy/handlers/anthropic.py`, `headroom/proxy/handlers/openai.py`: call `resolve_markers_in_response` on the finalized response JSON, right after existing CCR tool-call handling, at all three non-streaming response sites (Anthropic Messages, OpenAI Chat Completions backend path, OpenAI Responses API). Streaming responses are out of scope for this PR, tracked as follow-up, noted in the module docstring's scope. ## Testing - [x] Added new tests - [x] All tests pass locally ``` $ python -m pytest tests/test_ccr_marker_resolution.py -q ============================= test session starts ============================= collected 6 items tests\test_ccr_marker_resolution.py ...... [100%] ============================== 6 passed in 0.45s ============================== $ python -m pytest tests/test_ccr.py tests/test_ccr_response_handler.py tests/test_ccr_response_handler_extra.py tests/test_ccr_response_handler_openai_responses.py tests/test_proxy/test_openai_responses_ccr.py tests/test_proxy/test_anthropic_ccr_raise.py -q ======================= 83 passed, 1 warning in 34.50s ======================== ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.13.11, local headroom repo (`G:\Programmi Aggiuntivi\headroom`) - Exact command / steps: `python -m pytest tests/test_ccr_marker_resolution.py tests/test_ccr.py tests/test_ccr_response_handler.py tests/test_ccr_response_handler_extra.py tests/test_ccr_response_handler_openai_responses.py tests/test_proxy/test_openai_responses_ccr.py tests/test_proxy/test_anthropic_ccr_raise.py -q` - Observed result: 89 passed, 0 failed (6 new + 83 existing CCR tests, no regressions). `ruff check`, `ruff format --check`, and `mypy --ignore-missing-imports` all clean on every changed/new file. - Not tested: the actual Docker Compose / LiteLLM guardrail deployment from the issue's repro steps (no such environment available here); streaming response paths (out of scope, see Changes Made). ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review --------- Co-authored-by: Claude Sonnet 5 --- headroom/ccr/marker_resolution.py | 83 +++++++++++ headroom/cli/proxy.py | 15 ++ headroom/proxy/handlers/anthropic.py | 21 +++ headroom/proxy/handlers/openai.py | 45 ++++++ headroom/proxy/models.py | 7 + headroom/settings_store.py | 15 ++ tests/test_ccr_inline_resolve_handlers.py | 173 ++++++++++++++++++++++ tests/test_ccr_marker_resolution.py | 99 +++++++++++++ 8 files changed, 458 insertions(+) create mode 100644 headroom/ccr/marker_resolution.py create mode 100644 tests/test_ccr_inline_resolve_handlers.py create mode 100644 tests/test_ccr_marker_resolution.py diff --git a/headroom/ccr/marker_resolution.py b/headroom/ccr/marker_resolution.py new file mode 100644 index 000000000..aef4a108f --- /dev/null +++ b/headroom/ccr/marker_resolution.py @@ -0,0 +1,83 @@ +"""Inline resolution of ``<>`` markers on the response path. + +Normal CCR resolution relies on the ``headroom_retrieve`` tool: a marker is +redeemed when the model calls the tool back. That path assumes there's a +subsequent turn in which the model *can* call it. Callers that never see an +injected tool at all — e.g. Headroom running as a LiteLLM guardrail/proxy hop +with no tool-call turn in between (#2509) — have no way to redeem a marker, +so it leaks through as raw text. + +This module provides an explicit, opt-in fallback (``--ccr-inline-resolve``): +scan the outgoing response for markers and substitute the original content +directly, instead of leaving the marker for the model to redeem later. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any + +from ..cache.compression_store import ( + CompressionStore, + format_retrieval_miss_detail, + get_compression_store, +) + +logger = logging.getLogger(__name__) + +# Matches the opaque-blob marker form `<>` (and the +# row-offload form `<>`) emitted by SmartCrusher. +# HASH is 12-24 hex chars; see headroom/ccr/tool_injection.py for the same +# constant used on the injection side. +_MARKER_RE = re.compile(r"<]*>>") + + +def resolve_markers_in_text(text: str, *, store: CompressionStore | None = None) -> str: + """Replace every ``<>`` marker in ``text`` with its original content. + + A miss (expired/evicted/unknown hash) can't be reported back to the + model on this path — there's no tool-call round-trip — so the marker is + left in place with the miss reason appended rather than raising. + """ + if "< str: + hash_key = match.group(1) + entry = resolved_store.retrieve(hash_key) + if entry is not None: + original = entry.original_content + return original if isinstance(original, str) else json.dumps(original) + + get_status = getattr(resolved_store, "get_entry_status", None) + status = get_status(hash_key, clean_expired=True) if callable(get_status) else None + detail = format_retrieval_miss_detail(status) if status else "entry not found" + logger.warning(f"CCR inline-resolve: marker {hash_key} unresolvable ({detail})") + return f"{match.group(0)} [unresolved: {detail}]" + + return _MARKER_RE.sub(_replace, text) + + +def resolve_markers_in_response(response: Any, *, store: CompressionStore | None = None) -> Any: + """Recursively resolve ``<>`` markers in every string field of a payload. + + Walks the full response structure rather than picking out + provider-specific fields (``content`` blocks, ``message.content``, + Responses-API ``output`` items, ...) so it stays correct regardless of + where a marker ends up, and doesn't need per-provider maintenance. + """ + resolved_store = store or get_compression_store() + if isinstance(response, str): + return resolve_markers_in_text(response, store=resolved_store) + if isinstance(response, list): + return [resolve_markers_in_response(item, store=resolved_store) for item in response] + if isinstance(response, dict): + return { + key: resolve_markers_in_response(value, store=resolved_store) + for key, value in response.items() + } + return response diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py index 30292a4e1..29e964abd 100644 --- a/headroom/cli/proxy.py +++ b/headroom/cli/proxy.py @@ -313,6 +313,19 @@ def dashboard(port: int, no_open: bool) -> None: "retrieval marker, so no MCP retrieve tool is needed. Env: HEADROOM_LOSSLESS=1." ), ) +@click.option( + "--ccr-inline-resolve", + is_flag=True, + envvar="HEADROOM_CCR_INLINE_RESOLVE", + help=( + "Resolve <> markers inline on the response path instead of " + "relying on the model to call headroom_retrieve. For callers with no " + "tool-call round-trip to redeem a marker (e.g. Headroom running as a " + "LiteLLM guardrail/proxy hop, see issue #2509). Applies to non-streaming " + "responses only. Off by default. " + "Env: HEADROOM_CCR_INLINE_RESOLVE." + ), +) @click.option( "--no-ccr-proactive-expansion", is_flag=True, @@ -934,6 +947,7 @@ def proxy( tpm: int | None, no_ccr: bool, lossless: bool, + ccr_inline_resolve: bool, no_ccr_proactive_expansion: bool, proxy_extension: tuple[str, ...], compressor: tuple[str, ...], @@ -1206,6 +1220,7 @@ def proxy( # CCR fully on. ccr_inject_tool=not no_ccr, ccr_inject_marker=not no_ccr, + ccr_resolve_markers_inline=ccr_inline_resolve, lossless=lossless, ccr_proactive_expansion=not no_ccr_proactive_expansion, # Flatten repeat-flag tuple AND any comma-separated values inside it. diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py index d19413ac0..5125c0a01 100644 --- a/headroom/proxy/handlers/anthropic.py +++ b/headroom/proxy/handlers/anthropic.py @@ -25,6 +25,7 @@ import httpx from headroom.agent_savings import proxy_pipeline_kwargs from headroom.ccr.context_tracker import looks_like_claude_code_compact_summary +from headroom.ccr.marker_resolution import resolve_markers_in_response from headroom.copilot_auth import build_copilot_upstream_url from headroom.pipeline import PipelineStage, summarize_routing_markers from headroom.proxy.auth_mode import ( @@ -3763,6 +3764,26 @@ class AnthropicHandlerMixin: if _compression_failed: response_headers["x-headroom-compression-failed"] = "true" + # Inline CCR marker resolution, non-streaming only. + # Deliberately OUTSIDE the has_ccr_tool_calls gate + # above: the callers this flag exists for (#2509, + # Headroom behind a LiteLLM guardrail hop) never get + # a headroom_retrieve tool-call turn, so gating on + # one makes the flag a no-op for its own use case. + # Runs before the security scan so the scanner sees + # the resolved text, not the marker. + if ( + getattr(self.config, "ccr_resolve_markers_inline", False) + and resp_json + and response.status_code == 200 + ): + resp_json = resolve_markers_in_response(resp_json) + response = httpx.Response( + status_code=200, + content=json.dumps(resp_json).encode(), + headers=response_headers, + ) + # Enterprise Security: scan response + de-anonymize. # Gate on a 200 upstream like the sibling CCR/cache/buffered # blocks below: without this, a non-2xx upstream (rate limit diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py index 218ef7ec6..c3e3ff0a3 100644 --- a/headroom/proxy/handlers/openai.py +++ b/headroom/proxy/handlers/openai.py @@ -45,6 +45,7 @@ if TYPE_CHECKING: import httpx from headroom.agent_savings import proxy_pipeline_kwargs +from headroom.ccr.marker_resolution import resolve_markers_in_response from headroom.config import unwrap_tool_call_name from headroom.copilot_auth import ( apply_copilot_api_auth, @@ -4201,6 +4202,19 @@ class OpenAIHandlerMixin: # feedback_no_silent_fallbacks.md. raise + # Inline marker resolution runs OUTSIDE the tool-call + # gate above: the callers this flag exists for (#2509, + # Headroom as a LiteLLM guardrail hop) never emit a + # headroom_retrieve tool call, so gating on one would + # make the flag a no-op for exactly its use case. + # Non-streaming responses only. + if ( + getattr(self.config, "ccr_resolve_markers_inline", False) + and backend_response.body + and backend_response.status_code == 200 + ): + backend_response.body = resolve_markers_in_response(backend_response.body) + # Extract usage from the FINAL backend body (after # any CCR resolution) so the prefix tracker counts # cache stats from the LAST upstream call. @@ -4785,6 +4799,22 @@ class OpenAIHandlerMixin: if _compression_failed: response_headers["x-headroom-compression-failed"] = "true" + # Inline marker resolution, non-streaming only. The direct + # path has no CCR tool-call handling at all, which is + # exactly the #2509 shape: markers would otherwise reach + # the client as raw text. + if ( + getattr(self.config, "ccr_resolve_markers_inline", False) + and resp_json + and response.status_code == 200 + ): + resolved_json = resolve_markers_in_response(resp_json) + return Response( + content=json.dumps(resolved_json).encode(), + status_code=response.status_code, + headers=response_headers, + ) + return Response( content=response.content, status_code=response.status_code, @@ -5957,6 +5987,21 @@ class OpenAIHandlerMixin: headers=sse_headers, ) + # Inline marker resolution, non-streaming only. Runs + # outside the has_ccr_tool_calls gate above on purpose: + # the #2509 case has no retrieve tool call at all. + if ( + getattr(self.config, "ccr_resolve_markers_inline", False) + and resp_json + and response.status_code == 200 + ): + resolved_json = resolve_markers_in_response(resp_json) + return Response( + content=json.dumps(resolved_json).encode(), + status_code=response.status_code, + headers=response_headers, + ) + return Response( content=response.content, status_code=response.status_code, diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py index 069867e2b..e44493bbe 100644 --- a/headroom/proxy/models.py +++ b/headroom/proxy/models.py @@ -186,6 +186,13 @@ class ProxyConfig: # markers can be toggled from the CLI (--no-ccr, which also drops the retrieve # tool). Threaded into the router in server.py; default preserves current behavior. ccr_inject_marker: bool = True + # Explicit opt-in fallback for callers with no path to redeem a marker via + # the headroom_retrieve tool (e.g. a LiteLLM guardrail/proxy hop with no + # tool-call turn — issue #2509). Off by default: guessing "this caller + # can't use tools" is fragile, so operators must opt in with + # --ccr-inline-resolve / HEADROOM_CCR_INLINE_RESOLVE. Applies to + # non-streaming responses only; buffered-CCR streaming is untouched. + ccr_resolve_markers_inline: bool = False # CCR Response Handling ccr_handle_responses: bool = True diff --git a/headroom/settings_store.py b/headroom/settings_store.py index b686d3b86..ad134ebc0 100644 --- a/headroom/settings_store.py +++ b/headroom/settings_store.py @@ -136,6 +136,21 @@ SETTINGS: tuple[SettingField, ...] = ( help="Disable CCR entirely (no markers, no injected retrieve tool).", tier="basic", ), + SettingField( + "HEADROOM_CCR_INLINE_RESOLVE", + "ccr_inline_resolve", + "Resolve CCR markers inline", + "Compression", + "bool", + default=False, + help=( + "Resolve <> markers on the response path instead of " + "relying on headroom_retrieve tool calls. For callers with no " + "tool-call round-trip (e.g. a LiteLLM guardrail/proxy hop). " + "Non-streaming responses only." + ), + tier="advanced", + ), # --- Limits --- SettingField( "HEADROOM_RPM", diff --git a/tests/test_ccr_inline_resolve_handlers.py b/tests/test_ccr_inline_resolve_handlers.py new file mode 100644 index 000000000..51cab54c6 --- /dev/null +++ b/tests/test_ccr_inline_resolve_handlers.py @@ -0,0 +1,173 @@ +"""Handler-level wiring for ``--ccr-inline-resolve`` (issue #2509). + +The pure-module tests in ``test_ccr_marker_resolution.py`` prove the +substitution logic. These tests prove the thing that actually broke: the +resolve call has to run on the response path *when the model never emitted a +``headroom_retrieve`` tool call at all*. That is the whole #2509 shape — +Headroom behind a LiteLLM guardrail hop with no tool-call turn — so any wiring +that sits behind a ``has_ccr_tool_calls`` gate is a no-op for its own use case. + +Every response fixture below therefore has zero tool calls. +""" + +from __future__ import annotations + +import pytest + +fastapi = pytest.importorskip("fastapi") +httpx = pytest.importorskip("httpx") + +from fastapi.testclient import TestClient # noqa: E402 + +from headroom.cache.compression_store import ( # noqa: E402 + get_compression_store, + reset_compression_store, +) +from headroom.proxy.server import ProxyConfig, create_app # noqa: E402 + +ORIGINAL = "the original uncompressed content" + + +@pytest.fixture(autouse=True) +def reset_store(): + reset_compression_store() + yield + reset_compression_store() + + +def _marker() -> str: + hash_key = get_compression_store().store( + original=ORIGINAL, + compressed="[]", + original_item_count=1, + compressed_item_count=0, + ) + return f"<>" + + +def _config(*, inline_resolve: bool) -> ProxyConfig: + # No backend -> the "Direct OpenAI API (no backend configured)" path. + return ProxyConfig( + optimize=False, + cache_enabled=False, + rate_limit_enabled=False, + ccr_resolve_markers_inline=inline_resolve, + ) + + +def _chat_response(marker: str) -> dict: + return { + "id": "chatcmpl-1", + "object": "chat.completion", + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": f"here it is: {marker}"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + +def _responses_response(marker: str) -> dict: + return { + "id": "resp_1", + "object": "response", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": f"here it is: {marker}"}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _anthropic_response(marker: str) -> dict: + return { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [{"type": "text", "text": f"here it is: {marker}"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + +def _run(config: ProxyConfig, path: str, body: dict, upstream: dict) -> httpx.Response: + async def fake_retry(method, url, headers, req_body, *args, **kwargs): + return httpx.Response(200, json=upstream, headers={"content-type": "application/json"}) + + app = create_app(config) + with TestClient(app) as client: + client.app.state.proxy._retry_request = fake_retry + return client.post( + path, + json=body, + headers={"Authorization": "Bearer test-key", "x-api-key": "test-key"}, + ) + + +@pytest.mark.parametrize("inline_resolve", [True, False]) +def test_openai_chat_direct_path(inline_resolve): + marker = _marker() + resp = _run( + _config(inline_resolve=inline_resolve), + "/v1/chat/completions", + {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], "stream": False}, + _chat_response(marker), + ) + + assert resp.status_code == 200, resp.text + text = resp.json()["choices"][0]["message"]["content"] + if inline_resolve: + assert text == f"here it is: {ORIGINAL}" + else: + assert text == f"here it is: {marker}" + + +@pytest.mark.parametrize("inline_resolve", [True, False]) +def test_openai_responses_path(inline_resolve): + marker = _marker() + resp = _run( + _config(inline_resolve=inline_resolve), + "/v1/responses", + {"model": "gpt-4o", "input": "hi", "stream": False}, + _responses_response(marker), + ) + + assert resp.status_code == 200, resp.text + text = resp.json()["output"][0]["content"][0]["text"] + if inline_resolve: + assert text == f"here it is: {ORIGINAL}" + else: + assert text == f"here it is: {marker}" + + +@pytest.mark.parametrize("inline_resolve", [True, False]) +def test_anthropic_messages_path(inline_resolve): + marker = _marker() + resp = _run( + _config(inline_resolve=inline_resolve), + "/v1/messages", + { + "model": "claude-sonnet-4-20250514", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "stream": False, + }, + _anthropic_response(marker), + ) + + assert resp.status_code == 200, resp.text + text = resp.json()["content"][0]["text"] + if inline_resolve: + assert text == f"here it is: {ORIGINAL}" + else: + assert text == f"here it is: {marker}" diff --git a/tests/test_ccr_marker_resolution.py b/tests/test_ccr_marker_resolution.py new file mode 100644 index 000000000..ab2609c3b --- /dev/null +++ b/tests/test_ccr_marker_resolution.py @@ -0,0 +1,99 @@ +"""Tests for inline <> marker resolution (issue #2509).""" + +from __future__ import annotations + +import json + +import pytest + +from headroom.cache.compression_store import get_compression_store, reset_compression_store +from headroom.ccr.marker_resolution import ( + resolve_markers_in_response, + resolve_markers_in_text, +) + + +@pytest.fixture(autouse=True) +def reset_store(): + reset_compression_store() + yield + reset_compression_store() + + +def _store_entry(original: str) -> str: + store = get_compression_store() + return store.store( + original=original, + compressed="[]", + original_item_count=1, + compressed_item_count=0, + ) + + +def test_resolve_markers_in_text_no_marker_is_noop(): + assert resolve_markers_in_text("plain text, no markers here") == "plain text, no markers here" + + +def test_resolve_markers_in_text_replaces_hit(): + hash_key = _store_entry("the original uncompressed content") + text = f"before <> after" + + resolved = resolve_markers_in_text(text) + + assert resolved == "before the original uncompressed content after" + + +def test_resolve_markers_in_text_replaces_multiple_hits(): + hash_a = _store_entry("AAA") + hash_b = _store_entry("BBB") + text = f"<> and <>" + + resolved = resolve_markers_in_text(text) + + assert resolved == "AAA and BBB" + + +def test_resolve_markers_in_text_json_array_original_content(): + store = get_compression_store() + hash_key = store.store( + original=json.dumps([1, 2, 3]), + compressed="[]", + original_item_count=3, + compressed_item_count=0, + ) + text = f"<>" + + resolved = resolve_markers_in_text(text) + + assert json.loads(resolved) == [1, 2, 3] + + +def test_resolve_markers_in_text_miss_leaves_marker_with_reason(): + text = "<>" + + resolved = resolve_markers_in_text(text) + + assert text in resolved + assert "[unresolved:" in resolved + + +def test_resolve_markers_in_response_walks_nested_structure(): + hash_key = _store_entry("full tool output") + response = { + "choices": [ + { + "message": { + "role": "assistant", + "content": f"here it is: <>", + } + } + ], + "unrelated": 42, + "nested": {"list": ["a", f"<>", "c"]}, + } + + resolved = resolve_markers_in_response(response) + + assert resolved["choices"][0]["message"]["content"] == "here it is: full tool output" + assert resolved["nested"]["list"] == ["a", "full tool output", "c"] + assert resolved["unrelated"] == 42