From 455f4f263ce2638ff387c52d42601760f5cf0784 Mon Sep 17 00:00:00 2001 From: Abhay Singh Date: Mon, 24 Aug 2026 00:21:33 +0530 Subject: [PATCH] fix(cache/semantic): don't semantic-match an empty query across contexts (#3226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description `SemanticCache.get()` matches on the **embedding of the last user message** whenever an `embedding_fn` is wired. That query is empty (`""`) for the overwhelming majority of agent/tool turns — a `tool_result` continuation carries no text block, so `SemanticCacheLayer._extract_query` returns `""`. A real sentence embedder maps `""` to a fixed **non-zero** vector, so every empty-query turn is ~identical to every other in embedding space. The exact `messages_hash` guard (correctly chosen so `"continue"`/`"yes"` turns in different contexts don't collide) is then bypassed by the semantic path: an empty-query request misses on its unique hash, falls through to embedding matching, and hits a **different conversation's** stored response. Reproduction (realistic embedder, non-zero for `""`): ```python c = SemanticCache(embedding_fn=embed) c.put(query="", response={"answer": "A"}, messages_hash="ctxA") # conversation A c.get(query="", messages_hash="ctxB") # conversation B, different context # -> returned A's response (cross-context false hit) ``` Measured on 330 real Claude Code transcripts (28,441 requests): **95.7% have an empty extracted query**, so this is the dominant case, not a corner case. The exact-hash path is unaffected; only the embedding-similarity path is. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `headroom/cache/semantic.py`: - `get()`: gate the semantic-similarity branch on `query.strip()` — an empty/blank query can only ever hit via its exact `messages_hash` (context-complete), never via embedding similarity. - `put()`: store no embedding for an empty/blank query, so such an entry is skipped by `_find_similar` (which ignores entries with no embedding) and can never be a match target. - `tests/test_cache/test_semantic.py`: added `test_empty_query_never_semantic_matches` (cross-context empty-query miss, exact-hash still hits, whitespace treated as empty). ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality ### Test Output ```text tests/test_cache/test_semantic.py -> 22 passed in 2.39s uvx ruff@0.16.2 check headroom/cache/semantic.py tests/test_cache/test_semantic.py -> All checks passed! uvx mypy@1.20.2 headroom/cache/semantic.py -> Success: no issues found in 1 source file ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.12.11, project venv, pytest 9.1.1, ruff 0.16.2 and mypy 1.20.2 via uvx. - Exact command / steps: before the fix, two different-context empty-query requests (`ctxA` then `ctxB`) returned `ctxA`'s response via the embedding path. After the fix, the second returns `None`, while `ctxA`'s own exact-hash lookup still returns its response, and a legitimate non-empty semantic hit (`"What is the weather today?"` -> `"How is the weather?"`) still works. - Observed result: empty/blank queries no longer semantic-match across contexts; exact-hash and non-empty semantic matching are unchanged. - Not tested: no live embedder model wired (the current client wires none — the embedding path is exercised with an injected `embedding_fn`, which is the documented usage). ## Runtime Rollout Safety - Rollout-managed feature(s): none. `SemanticCache` is an SDK-side cache (`headroom.cache`), not a rollout-channel-gated runtime feature; semantic matching only runs when a caller injects an `embedding_fn`. - Minimum rollout channel: N/A. - Stable/default behavior changed: no. Exact-hash matching and non-empty semantic matching are unchanged; only empty/blank-query semantic matching (a false-hit source) is removed. - Kill switch / disable path: N/A. - Unsafe override required: no. - Qualification impact: none; correctness-only. - Rollback path: revert this PR. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation (N/A: internal behavior) - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` --- headroom/cache/semantic.py | 22 ++++++++++++++++++---- tests/test_cache/test_semantic.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.py index 0ea2effba..946fada32 100644 --- a/headroom/cache/semantic.py +++ b/headroom/cache/semantic.py @@ -160,8 +160,18 @@ class SemanticCache: self._hits += 1 return entry - # Try semantic similarity if we have embedding function - if self._embedding_fn: + # Try semantic similarity if we have embedding function. + # + # Only for a NON-EMPTY query: the query is the last user message, and in + # agent/tool traffic the overwhelming majority of turns are tool_result + # continuations whose extracted query is "" (no text block). Embedding + # matching on "" makes every such turn ~identical to every other (a real + # sentence embedder maps "" to a fixed non-zero vector), so an empty + # query would false-hit and serve one conversation's response to an + # unrelated one — precisely the cross-context collision the messages_hash + # key is chosen to avoid. An empty query may still hit via the exact + # messages_hash above, which is context-complete and safe. + if self._embedding_fn and query.strip(): query_embedding = self._embedding_fn(query) best_match, best_similarity = self._find_similar(query_embedding) @@ -208,9 +218,13 @@ class SemanticCache: while key not in self._cache and len(self._cache) >= self.config.max_entries: self._evict_oldest() - # Generate embedding if available + # Generate embedding if available — but never for an empty/blank query. + # A stored empty-query entry with an embedding would be a false-match + # target for the semantic get() path; leaving its embedding empty makes + # _find_similar skip it (it ignores entries with no embedding), so an + # empty-query entry is reachable only by its exact messages_hash. embedding: list[float] = [] - if self._embedding_fn: + if self._embedding_fn and query.strip(): embedding = self._embedding_fn(query) now = time.time() diff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.py index c24d261f9..f5923f1cb 100644 --- a/tests/test_cache/test_semantic.py +++ b/tests/test_cache/test_semantic.py @@ -208,6 +208,36 @@ class TestSemanticCache: entry = cache.get("What time is it?") assert entry is None + def test_empty_query_never_semantic_matches(self): + """An empty extracted query must not trigger cross-context false hits. + + The query is the last user message; in agent/tool traffic most turns are + tool_result continuations whose extracted query is "". A real embedder + maps "" to a fixed non-zero vector, so without a guard every empty-query + turn would be ~identical to every other and serve one conversation's + response to an unrelated one. An empty query may only ever hit via the + exact messages_hash (which is context-complete). + """ + + def const_embedding(text: str) -> list[float]: + # Realistic: a non-zero, identical vector for every input (incl. ""). + return [0.5, 0.5, 0.5] + + config = SemanticCacheConfig(similarity_threshold=0.9) + cache = SemanticCache(config, embedding_fn=const_embedding) + + # Conversation A: an empty-query turn (unique full-context hash). + cache.put("", "response-A", messages_hash="ctxA") + + # Conversation B: a different empty-query turn — must NOT get A's answer. + assert cache.get("", messages_hash="ctxB") is None + # Its own exact hash still works. + assert cache.get("", messages_hash="ctxA").response == "response-A" + + # A whitespace-only query is treated the same as empty. + cache.put(" \n\t", "response-C", messages_hash="ctxC") + assert cache.get(" ", messages_hash="ctxD") is None + class TestSemanticCacheLayer: """Test SemanticCacheLayer functionality."""