diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a2ca1f6..9b886d576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Bug Fixes +* **proxy:** stop discarding a finished compression on very large requests. After the transform pipeline completed, a telemetry-only waste-signal re-parse of the *original* messages ran on the critical path; on huge Claude Code transcripts (~400k tokens) that parse could exceed the Anthropic compression timeout, so the proxy failed open and forwarded the uncompressed request despite "Pipeline complete" logging real savings (`tokens_saved: 0`, `transforms_applied: []`, ~31s latency). Waste-signal detection is now skipped above `MAX_WASTE_SIGNAL_DETECTION_TOKENS` (100k) so the compression result stays on the critical path ([#296](https://github.com/chopratejas/headroom/issues/296)). * **codex:** retag existing Codex threads when `headroom init` injects the `headroom` provider, so Codex Desktop history stays visible. Codex filters its sidebar/search by the active `model_provider`; the init path set `model_provider = "headroom"` without retagging, so existing native `openai` threads disappeared from the menu (data was never deleted, only hidden). `_ensure_codex_provider` now reconciles thread tags openai→headroom, matching what the install and `wrap` paths already do; `headroom unwrap codex` handles the revert direction ([#961](https://github.com/chopratejas/headroom/issues/961)). * **install:** stop duplicating the container ENTRYPOINT in the `persistent-docker` runtime command. The published image already runs `headroom proxy` as its ENTRYPOINT, but `build_runtime_command` re-added `headroom proxy` after the image name, so the container ran `headroom proxy headroom proxy --host 0.0.0.0 …` and Click aborted with "Got unexpected extra arguments (headroom proxy)" — the deployment never became ready and rollback left nothing running. The runtime command now appends only the proxy flags ([#833](https://github.com/chopratejas/headroom/issues/833)). * **code:** keep Python `from __future__` imports before executable code during AST compression and validate compressed Python with `compile(..., "exec")` so compile-time syntax rules are enforced ([#1233](https://github.com/chopratejas/headroom/issues/1233)). diff --git a/headroom/transforms/pipeline.py b/headroom/transforms/pipeline.py index 5447ea741..fec4701a2 100644 --- a/headroom/transforms/pipeline.py +++ b/headroom/transforms/pipeline.py @@ -30,6 +30,18 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Waste-signal detection re-parses the *original* messages for telemetry only +# (it never changes the compression result). On very large transcripts that +# extra parse can take tens of seconds and blow the Anthropic compression +# timeout, making the proxy fail open and discard an already-computed +# compression (#296). Skip the diagnostic above this size to keep the result +# on the critical path. +MAX_WASTE_SIGNAL_DETECTION_TOKENS = 100_000 + +# A token saving below this is treated as noise — waste-signal detection only +# runs when compression saved more than this many tokens. +_MIN_TOKENS_SAVED_FOR_WASTE_SIGNALS = 100 + _N = TypeVar("_N", int, float) @@ -220,6 +232,9 @@ class TransformPipeline: """ record_metrics = kwargs.pop("record_metrics", True) waste_messages = kwargs.pop("waste_messages", None) + waste_signal_token_limit = int( + kwargs.pop("waste_signal_token_limit", MAX_WASTE_SIGNAL_DETECTION_TOKENS) + ) tokenizer = self._get_tokenizer(model) provider_name = self._provider_name() @@ -439,7 +454,21 @@ class TransformPipeline: # pass a richer waste_messages list that is parsed instead — it is # telemetry-only and never transformed. waste_signals: WasteSignals | None = None - if tokens_before > tokens_after and (tokens_before - tokens_after) > 100: + saved_enough = ( + tokens_before > tokens_after + and (tokens_before - tokens_after) > _MIN_TOKENS_SAVED_FOR_WASTE_SIGNALS + ) + if saved_enough and tokens_before > waste_signal_token_limit: + # Telemetry-only re-parse would risk the compression timeout on a + # request this large (#296); skip it and keep the result. + logger.debug( + "%sSkipping waste-signal detection for %d-token request " + "(limit=%d) to keep the compression result on the critical path", + log_prefix, + tokens_before, + waste_signal_token_limit, + ) + elif saved_enough: try: from ..parser import parse_messages diff --git a/tests/test_transforms/test_pipeline_waste_signal_limit.py b/tests/test_transforms/test_pipeline_waste_signal_limit.py new file mode 100644 index 000000000..b578980f3 --- /dev/null +++ b/tests/test_transforms/test_pipeline_waste_signal_limit.py @@ -0,0 +1,95 @@ +"""Waste-signal detection must not discard a finished compression (#296). + +On very large Claude Code transcripts the telemetry-only waste-signal re-parse +of the *original* messages can take tens of seconds and blow the Anthropic +compression timeout, making the proxy fail open and forward the original +request even though compression already succeeded. The pipeline now skips that +diagnostic above ``MAX_WASTE_SIGNAL_DETECTION_TOKENS`` so the compression +result stays on the critical path. +""" + +from __future__ import annotations + +from typing import Any + +from headroom.config import HeadroomConfig, TransformResult +from headroom.transforms.base import Transform +from headroom.transforms.pipeline import TransformPipeline + + +class _FakeTokenizer: + """Reports a fixed token count for the original messages so the test can + drive ``tokens_before`` above or below the waste-signal limit.""" + + def __init__(self, before: int, after: int) -> None: + self._before = before + self._after = after + + def count_messages(self, messages: list[dict[str, Any]]) -> int: + # The compressed message carries the marker "compressed". + if any(m.get("content") == "compressed" for m in messages): + return self._after + return self._before + + def count_text(self, text: Any) -> int: + return len(str(text)) + + +class _ShrinkTransform(Transform): + name = "test_shrink" + + def apply( + self, messages: list[dict[str, Any]], tokenizer: Any, **kwargs: Any + ) -> TransformResult: + optimized = [dict(m) for m in messages] + optimized[-1] = {**optimized[-1], "content": "compressed"} + return TransformResult( + messages=optimized, + tokens_before=tokenizer.count_messages(messages), + tokens_after=tokenizer.count_messages(optimized), + transforms_applied=["test:shrink"], + ) + + +def _run(monkeypatch, *, before: int, after: int, limit: int): + """Run the pipeline with a stub transform; return (result, parse_called).""" + pipeline = TransformPipeline(HeadroomConfig()) + pipeline.transforms = [_ShrinkTransform()] + monkeypatch.setattr(pipeline, "_get_tokenizer", lambda _model: _FakeTokenizer(before, after)) + + parse_called = False + + def _tracked_parse_messages(*args: Any, **kwargs: Any): + nonlocal parse_called + parse_called = True + return [], {}, None + + monkeypatch.setattr("headroom.parser.parse_messages", _tracked_parse_messages) + + messages = [{"role": "user", "content": "x" * 1000}] + result = pipeline.apply( + messages, + model="claude-3-5-sonnet", + model_limit=1_000_000, + record_metrics=False, + waste_signal_token_limit=limit, + ) + return result, parse_called + + +def test_large_request_skips_waste_signal_and_keeps_compression(monkeypatch): + """Above the limit, waste-signal detection is skipped but the compression + result is preserved (the bug discarded it via the timeout).""" + result, parse_called = _run(monkeypatch, before=200_000, after=180_000, limit=100_000) + + assert parse_called is False, "waste-signal parse must be skipped above the limit" + assert "test:shrink" in result.transforms_applied + assert result.tokens_after < result.tokens_before + assert result.messages[-1]["content"] == "compressed" + + +def test_small_request_still_runs_waste_signal_detection(monkeypatch): + """Below the limit, the diagnostic still runs (no behavior change).""" + _result, parse_called = _run(monkeypatch, before=10_000, after=5_000, limit=100_000) + + assert parse_called is True, "waste-signal parse must still run below the limit"