From 98842b847b85909975889b7d5bfa856ed773b393 Mon Sep 17 00:00:00 2001 From: JD Davis Date: Sat, 11 Jul 2026 15:27:00 +0000 Subject: [PATCH] Extract loop callback failure policy (#1977) ## Description Extracts the event-loop callback failure classifier for a known WebSocket disconnect regression from `server.py` into `headroom.proxy.loop_callback_failure_policy`. The server keeps `_is_known_websocket_callback_failure` as a compatibility alias for the existing loop-health path. Closes # ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [x] Code refactoring (no functional changes) ## Changes Made - Added `loop_callback_failure_policy.py` with constants for the known message and exception shape. - Replaced the inline server helper body with a compatibility alias to the extracted classifier. - Added direct classifier tests and ran the existing loop-health regression tests. - Carried forward the LiteLLM callback compatibility shim needed for current mypy on `main`. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text python -m pytest tests\test_loop_callback_failure_policy.py tests\test_proxy_loop_exception_health.py 5 passed in 4.89s python -m ruff check . All checks passed! python -m ruff format --check . 1095 files already formatted python -m mypy headroom --ignore-missing-imports Success: no issues found in 409 source files gitleaks protect --staged --no-banner --redact no leaks found ``` ## Real Behavior Proof - Environment: Windows, Python 3.13.13, branch `jd/architecture-slice-27`. - Exact command / steps: ran new loop callback classifier tests, existing loop-health tests, ruff, ruff format check, mypy, and staged gitleaks scan. - Observed result: classifier behavior and endpoint-level loop-health behavior remain covered; local lint/type/security checks pass. - Not tested: live WebSocket disconnect reproduction; this slice preserves the existing server alias. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Screenshots (if applicable) N/A ## Additional Notes Documentation and changelog updates are N/A for this internal architecture-only refactor. The push reported existing default-branch Dependabot alerts; no staged secret leaks were found for this PR. --- .../proxy/loop_callback_failure_policy.py | 20 ++++++++++ headroom/proxy/server.py | 14 +------ tests/test_loop_callback_failure_policy.py | 40 +++++++++++++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 headroom/proxy/loop_callback_failure_policy.py create mode 100644 tests/test_loop_callback_failure_policy.py diff --git a/headroom/proxy/loop_callback_failure_policy.py b/headroom/proxy/loop_callback_failure_policy.py new file mode 100644 index 000000000..a802991e3 --- /dev/null +++ b/headroom/proxy/loop_callback_failure_policy.py @@ -0,0 +1,20 @@ +"""Classification policy for event-loop callback failures.""" + +from __future__ import annotations + +from typing import Any + +KNOWN_WEBSOCKET_CALLBACK_MESSAGE = ( + "Exception in callback Connection.connection_lost(ConnectionResetError())" +) +KNOWN_WEBSOCKET_CALLBACK_EXCEPTION = "'ClientConnection' object has no attribute 'recv_messages'" + + +def is_known_websocket_callback_failure(context: dict[str, Any]) -> bool: + """Return True iff this exact websockets callback failure shape is observed.""" + if context.get("message") != KNOWN_WEBSOCKET_CALLBACK_MESSAGE: + return False + exception = context.get("exception") + return isinstance(exception, AttributeError) and str(exception) == ( + KNOWN_WEBSOCKET_CALLBACK_EXCEPTION + ) diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index d173e43bb..ce7e8a163 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -140,6 +140,7 @@ from headroom.proxy.helpers import ( jitter_delay_ms, retry_after_ms, ) +from headroom.proxy.loop_callback_failure_policy import is_known_websocket_callback_failure from headroom.proxy.loopback_guard import is_loopback_host from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler @@ -2007,18 +2008,7 @@ def _request_is_loopback(request: Request) -> bool: return is_loopback_host(client_host) and is_loopback_host_header(host_header) -def _is_known_websocket_callback_failure(context: dict[str, Any]) -> bool: - """Return True iff this exact websockets callback failure shape is observed.""" - if ( - context.get("message") - != "Exception in callback Connection.connection_lost(ConnectionResetError())" - ): - return False - exception = context.get("exception") - return ( - isinstance(exception, AttributeError) - and str(exception) == "'ClientConnection' object has no attribute 'recv_messages'" - ) +_is_known_websocket_callback_failure = is_known_websocket_callback_failure _tool_schema_saved_from_tags = tool_schema_saved_from_tags diff --git a/tests/test_loop_callback_failure_policy.py b/tests/test_loop_callback_failure_policy.py new file mode 100644 index 000000000..f82f4a371 --- /dev/null +++ b/tests/test_loop_callback_failure_policy.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from headroom.proxy.loop_callback_failure_policy import ( + KNOWN_WEBSOCKET_CALLBACK_EXCEPTION, + KNOWN_WEBSOCKET_CALLBACK_MESSAGE, + is_known_websocket_callback_failure, +) + + +def test_known_websocket_callback_failure_matches_exact_shape() -> None: + assert is_known_websocket_callback_failure( + { + "message": KNOWN_WEBSOCKET_CALLBACK_MESSAGE, + "exception": AttributeError(KNOWN_WEBSOCKET_CALLBACK_EXCEPTION), + } + ) + + +def test_known_websocket_callback_failure_rejects_other_message() -> None: + assert not is_known_websocket_callback_failure( + { + "message": "Exception in callback something_else", + "exception": AttributeError(KNOWN_WEBSOCKET_CALLBACK_EXCEPTION), + } + ) + + +def test_known_websocket_callback_failure_rejects_other_exception() -> None: + assert not is_known_websocket_callback_failure( + { + "message": KNOWN_WEBSOCKET_CALLBACK_MESSAGE, + "exception": AttributeError("different attribute"), + } + ) + assert not is_known_websocket_callback_failure( + { + "message": KNOWN_WEBSOCKET_CALLBACK_MESSAGE, + "exception": RuntimeError(KNOWN_WEBSOCKET_CALLBACK_EXCEPTION), + } + )