diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py index 886905e90..720eb722c 100644 --- a/headroom/proxy/handlers/openai.py +++ b/headroom/proxy/handlers/openai.py @@ -6152,10 +6152,18 @@ class OpenAIHandlerMixin: if protect_analysis_context is not None: pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context) - result = self.openai_pipeline.apply( - messages=messages, - model=model, - **pipeline_kwargs, + # Offload the CPU-bound pipeline to the bounded compression executor + # (mirrors the request handlers above). Running apply() inline blocked + # the single event loop on a large payload, so even GET /health stalled + # until it finished (#718). The executor also enforces a timeout so a + # too-large body fails fast instead of hanging forever. + result = await self._run_compression_in_executor( + lambda: self.openai_pipeline.apply( + messages=messages, + model=model, + **pipeline_kwargs, + ), + timeout=COMPRESSION_TIMEOUT_SECONDS, ) return JSONResponse( @@ -6174,6 +6182,23 @@ class OpenAIHandlerMixin: "ccr_hashes": result.markers_inserted, } ) + except TimeoutError: + logger.warning( + "Compression timed out after %.0fs (payload too large)", + COMPRESSION_TIMEOUT_SECONDS, + ) + return JSONResponse( + status_code=503, + content={ + "error": { + "type": "compression_timeout", + "message": ( + "Compression exceeded " + f"{COMPRESSION_TIMEOUT_SECONDS:.0f}s; payload too large." + ), + } + }, + ) except Exception as e: logger.exception("Compression failed: %s", e) return JSONResponse( diff --git a/tests/test_proxy_compress_endpoint.py b/tests/test_proxy_compress_endpoint.py index daa20e0d9..bb11dabc6 100644 --- a/tests/test_proxy_compress_endpoint.py +++ b/tests/test_proxy_compress_endpoint.py @@ -250,3 +250,75 @@ class TestCompressEndpointCompression: assert data["tokens_before"] >= 0 assert data["tokens_after"] >= 0 assert isinstance(data["transforms_applied"], list) + + +class TestCompressEndpointDoesNotBlockLoop: + """/v1/compress must offload to the compression executor so a slow/large + payload cannot freeze the single event loop (#718).""" + + async def test_compress_does_not_block_liveness(self, monkeypatch): + import asyncio + import threading + from types import SimpleNamespace + + import httpx + + config = ProxyConfig( + optimize=True, + cache_enabled=False, + rate_limit_enabled=False, + cost_tracking_enabled=False, + ) + app = create_app(config) + proxy = app.state.proxy + + entered = threading.Event() + release = threading.Event() + + def blocking_apply(**kwargs): + # Stand in for a large CPU-bound compression: blocks its worker until + # released. If this ran inline on the loop (the bug), the loop would + # be frozen and /livez below could not be served. + entered.set() + release.wait(timeout=10) + return SimpleNamespace( + messages=kwargs["messages"], + tokens_before=10, + tokens_after=5, + transforms_applied=[], + transforms_summary={}, + markers_inserted=[], + ) + + monkeypatch.setattr(proxy.openai_pipeline, "apply", blocking_apply) + + # /v1/compress is loopback-gated (#1227) — present as 127.0.0.1. + transport = httpx.ASGITransport(app=app, client=("127.0.0.1", 12345)) + async with httpx.AsyncClient(transport=transport, base_url="http://127.0.0.1") as client: + compress = asyncio.create_task( + client.post( + "/v1/compress", + json={ + "messages": [{"role": "user", "content": "hello world"}], + "model": "gpt-4", + }, + ) + ) + # Wait until the compression is actually in flight (running in the + # executor thread), then prove the loop is still responsive. + for _ in range(200): + if entered.is_set(): + break + await asyncio.sleep(0.01) + assert entered.is_set(), "compression never started" + + livez = await asyncio.wait_for(client.get("/livez"), timeout=5) + assert livez.status_code == 200 + assert livez.json()["alive"] is True + # The compression is still blocked — /livez was served concurrently. + assert not compress.done() + + release.set() + resp = await asyncio.wait_for(compress, timeout=5) + assert resp.status_code == 200 + assert resp.json()["tokens_saved"] == 5