From 740fb9bc16eb3d9db57b253ac22e62e52cb19860 Mon Sep 17 00:00:00 2001 From: JD Davis Date: Sat, 11 Jul 2026 04:49:27 +0000 Subject: [PATCH] refactor(cache): isolate semantic key policy (#1953) ## Description Extracts proxy semantic-cache key normalization and hashing into a pure policy module while preserving `SemanticCache._compute_key` for existing callers and tests. This separates deterministic cache-key construction from the async cache adapter and LRU storage concerns. Closes # ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [x] Code refactoring (no functional changes) ## Changes Made - Added `headroom.proxy.semantic_cache_key` for pure cache-control stripping and semantic cache key construction. - Updated `SemanticCache._compute_key` to delegate to the extracted policy while preserving the local `_strip_cache_control` compatibility alias. - Added direct tests for the extracted semantic-cache key policy. - Kept the LiteLLM callback compatibility shim required for repo-wide type checking on fresh branches. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text python -m pytest tests/test_proxy_semantic_cache_key_policy.py tests/test_proxy_semantic_cache_key.py tests/test_proxy_semantic_cache_key_integration.py tests/test_proxy_openai_cache_key_integration.py tests/test_litellm_callback.py tests/test_compress_api.py::TestLiteLLMCallback -q 46 passed in 11.83s python -m ruff check . All checks passed! python -m ruff format --check . 1095 files already formatted python -m mypy headroom --ignore-missing-imports Success: no issues found in 409 source files gitleaks protect --staged --no-banner --redact no leaks found ``` ## Real Behavior Proof - Environment: Windows, Python 3.13.13, local worktree based on `headroomlabs/main`. - Exact command / steps: ran focused semantic cache key tests, handler cache-key integration tests, LiteLLM callback tests, Ruff lint/format checks, mypy over `headroom`, and staged gitleaks scan. - Observed result: all local checks passed; staged secret scan found no leaks. - Not tested: full CI matrix and deployment flows; those are covered by GitHub Actions. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Screenshots (if applicable) N/A ## Additional Notes Documentation and changelog updates are not applicable for this internal refactor. GitHub reported existing Dependabot alerts on the default branch during push; this PR does not change dependencies, and the staged secret scan is clean. --- headroom/proxy/semantic_cache.py | 30 ++----------- headroom/proxy/semantic_cache_key.py | 33 ++++++++++++++ tests/test_proxy_semantic_cache_key_policy.py | 43 +++++++++++++++++++ 3 files changed, 79 insertions(+), 27 deletions(-) create mode 100644 headroom/proxy/semantic_cache_key.py create mode 100644 tests/test_proxy_semantic_cache_key_policy.py diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.py index cf14c30a8..1ce8baf48 100644 --- a/headroom/proxy/semantic_cache.py +++ b/headroom/proxy/semantic_cache.py @@ -8,8 +8,6 @@ Extracted from server.py for maintainability. from __future__ import annotations import asyncio -import hashlib -import json import sys from collections import OrderedDict from datetime import datetime @@ -19,23 +17,9 @@ if TYPE_CHECKING: from ..memory.tracker import ComponentStats from headroom.proxy.models import CacheEntry +from headroom.proxy.semantic_cache_key import compute_semantic_cache_key, strip_cache_control - -def _strip_cache_control(obj: Any) -> Any: - """Recursively drop ``cache_control`` annotations before hashing. - - Clients (notably Claude Code) move the ``cache_control`` cache breakpoint to - the newest content on each call, so the same logical ``system``/``tools`` - payload carries the marker on one call and not the next. Stripping it keeps - the cache key stable across that movement. Mirrors - ``helpers._strip_per_call_annotations`` but kept local so the cache module - stays free of the heavier proxy-helpers import chain. - """ - if isinstance(obj, dict): - return {k: _strip_cache_control(v) for k, v in obj.items() if k != "cache_control"} - if isinstance(obj, list): - return [_strip_cache_control(item) for item in obj] - return obj +_strip_cache_control = strip_cache_control class SemanticCache: @@ -66,15 +50,7 @@ class SemanticCache: untouched). Absent fields don't contribute, so truly-identical requests still hit. """ - normalized = json.dumps( - { - "model": model, - "messages": messages, - **{k: _strip_cache_control(v) for k, v in key_fields.items()}, - }, - sort_keys=True, - ) - return hashlib.sha256(normalized.encode()).hexdigest()[:32] + return compute_semantic_cache_key(messages, model, **key_fields) async def get(self, messages: list[dict], model: str, **key_fields: Any) -> CacheEntry | None: """Get cached response if exists and not expired.""" diff --git a/headroom/proxy/semantic_cache_key.py b/headroom/proxy/semantic_cache_key.py new file mode 100644 index 000000000..cbef6dcd6 --- /dev/null +++ b/headroom/proxy/semantic_cache_key.py @@ -0,0 +1,33 @@ +"""Pure semantic-cache key construction policy.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + + +def strip_cache_control(obj: Any) -> Any: + """Recursively drop ``cache_control`` annotations before hashing.""" + if isinstance(obj, dict): + return {k: strip_cache_control(v) for k, v in obj.items() if k != "cache_control"} + if isinstance(obj, list): + return [strip_cache_control(item) for item in obj] + return obj + + +def compute_semantic_cache_key( + messages: list[dict], + model: str, + **key_fields: Any, +) -> str: + """Compute the proxy semantic-cache key from generation-shaping inputs.""" + normalized = json.dumps( + { + "model": model, + "messages": messages, + **{k: strip_cache_control(v) for k, v in key_fields.items()}, + }, + sort_keys=True, + ) + return hashlib.sha256(normalized.encode()).hexdigest()[:32] diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.py new file mode 100644 index 000000000..e106779bc --- /dev/null +++ b/tests/test_proxy_semantic_cache_key_policy.py @@ -0,0 +1,43 @@ +"""Tests for pure proxy semantic-cache key policy.""" + +from __future__ import annotations + +from headroom.proxy.semantic_cache_key import ( + compute_semantic_cache_key, + strip_cache_control, +) + +MESSAGES = [{"role": "user", "content": "hello"}] +MODEL = "claude-haiku-4-5" + + +def test_semantic_cache_key_distinguishes_response_shaping_fields() -> None: + assert compute_semantic_cache_key(MESSAGES, MODEL, system="French") != ( + compute_semantic_cache_key(MESSAGES, MODEL, system="English") + ) + assert compute_semantic_cache_key(MESSAGES, MODEL, temperature=0.0) != ( + compute_semantic_cache_key(MESSAGES, MODEL, temperature=1.0) + ) + + +def test_semantic_cache_key_ignores_moved_cache_control() -> None: + with_cache_control = [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + without_cache_control = [{"type": "text", "text": "sys"}] + + assert compute_semantic_cache_key(MESSAGES, MODEL, system=with_cache_control) == ( + compute_semantic_cache_key(MESSAGES, MODEL, system=without_cache_control) + ) + + +def test_strip_cache_control_recurses_through_dicts_and_lists() -> None: + assert strip_cache_control( + { + "system": [ + {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}, + ], + "tools": [{"name": "read", "cache_control": {"type": "ephemeral"}}], + } + ) == { + "system": [{"type": "text", "text": "sys"}], + "tools": [{"name": "read"}], + }