mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-10 14:27:00 -04:00
c2fc4d3753
The optional `query` parameter on headroom_retrieve routed retrieval through CompressionStore.search(), which BM25-scored the items inside a single cached blob and dropped everything below a 0.3 relevance floor. On small per-blob corpora with conversational queries this returned an empty result the large majority of the time, so the LLM saw "nothing found" for content that was actually present — pushing users to turn compression off entirely. Retrieval is fundamentally a hash lookup (this already matches the Rust proxy's CCR store, which is put/get only — "no BM25 search"). Remove the query/search path end to end and always return the full original content: Core (Python proxy): - tool schemas (anthropic/openai/google) drop the `query` property - parse_tool_call returns the hash (str | None) instead of (hash, query) - response handler, proxy POST/GET/tool-call handlers, the MCP retrieve tool, and the streaming feedback recorders retrieve by hash only - proactive context-tracker expansion always restores full content - delete CompressionStore.search() and its BM25 machinery (the bm25 module stays — it is still used by relevance/) - CCRToolCall.query, CCRToolResult.was_search, and ExpansionRecommendation.expand_full/search_query are removed Plugins (advertised a now-defunct query param to the LLM): - hermes (Python), openclaw + opencode (TypeScript) retrieve tools drop `query` from their schemas, signatures, request URLs, and tests Benchmarks/docs: - ccr_regression + adversarial benchmarks switch from store.search() to full hash retrieval (search input-injection tests repurposed to the hash, the only remaining input surface) - wiki/ARCHITECTURE.md, wiki/ccr.md, docs/content/docs/ccr.mdx, config.py and store docstrings updated to describe hash-only retrieval Tests updated to assert full-content retrieval and guard the removed surface; the full CCR/proxy/store/TOIN suite passes. ruff + mypy clean. ## Description <!-- Briefly explain the change and why it is needed. --> Closes # ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - ## Testing <!-- Check what you actually ran, then paste the real command output below. --> - [ ] Unit tests pass (`pytest`) - [ ] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [ ] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text # Paste relevant command output or artifact links here ``` ## Real Behavior Proof - Environment: - Exact command / steps: - Observed result: - Not tested: ## Review Readiness - [ ] I have performed a self-review - [ ] This PR is ready for human review ## Checklist - [ ] My code follows the project's style guidelines - [ ] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Screenshots (if applicable) Add screenshots to help explain your changes. ## Additional Notes <!-- Mention any N/A checklist items, tradeoffs, follow-ups, or maintainer context. -->
355 lines
12 KiB
Python
355 lines
12 KiB
Python
"""Tests for CCR feedback loop and pattern learning."""
|
|
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from headroom.cache.compression_feedback import (
|
|
CompressionFeedback,
|
|
LocalToolPattern,
|
|
get_compression_feedback,
|
|
reset_compression_feedback,
|
|
)
|
|
from headroom.cache.compression_store import (
|
|
CompressionStore,
|
|
RetrievalEvent,
|
|
reset_compression_store,
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_globals():
|
|
"""Reset global state before each test."""
|
|
reset_compression_feedback()
|
|
reset_compression_store()
|
|
yield
|
|
reset_compression_feedback()
|
|
reset_compression_store()
|
|
|
|
|
|
class TestCompressionFeedback:
|
|
"""Test CompressionFeedback analyzer."""
|
|
|
|
def test_record_compression(self):
|
|
"""Recording compression events updates tool patterns."""
|
|
feedback = CompressionFeedback()
|
|
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
feedback.record_compression("test_tool", 200, 20)
|
|
|
|
patterns = feedback.get_all_patterns()
|
|
assert "test_tool" in patterns
|
|
assert patterns["test_tool"].total_compressions == 2
|
|
|
|
def test_record_retrieval(self):
|
|
"""Recording retrieval events updates patterns."""
|
|
feedback = CompressionFeedback()
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query="find errors",
|
|
items_retrieved=50,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="search",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
patterns = feedback.get_all_patterns()
|
|
assert patterns["test_tool"].total_retrievals == 1
|
|
assert patterns["test_tool"].search_retrievals == 1
|
|
|
|
def test_retrieval_rate_calculation(self):
|
|
"""Retrieval rate is calculated correctly."""
|
|
feedback = CompressionFeedback()
|
|
|
|
# 10 compressions
|
|
for _ in range(10):
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
# 5 retrievals (50% retrieval rate)
|
|
for _ in range(5):
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query=None,
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="full",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
pattern = feedback.get_all_patterns()["test_tool"]
|
|
assert pattern.retrieval_rate == 0.5
|
|
assert pattern.full_retrieval_rate == 1.0 # All were full retrievals
|
|
|
|
def test_hints_default_with_no_data(self):
|
|
"""Default hints returned when no data exists."""
|
|
feedback = CompressionFeedback()
|
|
|
|
hints = feedback.get_compression_hints("unknown_tool")
|
|
|
|
assert hints.max_items == 15 # Default
|
|
assert hints.skip_compression is False
|
|
assert "No pattern data" in hints.reason
|
|
|
|
def test_hints_insufficient_samples(self):
|
|
"""Default hints returned with insufficient samples."""
|
|
feedback = CompressionFeedback()
|
|
|
|
# Only 3 compressions (need 5 for hints)
|
|
for _ in range(3):
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
hints = feedback.get_compression_hints("test_tool")
|
|
|
|
assert hints.max_items == 15 # Default
|
|
assert "Insufficient data" in hints.reason
|
|
|
|
def test_hints_high_retrieval_rate_less_aggressive(self):
|
|
"""High retrieval rate results in less aggressive compression."""
|
|
feedback = CompressionFeedback()
|
|
|
|
# 10 compressions
|
|
for _ in range(10):
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
# 6 retrievals (60% retrieval rate - HIGH)
|
|
for _ in range(6):
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query="search query",
|
|
items_retrieved=50,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="search",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
hints = feedback.get_compression_hints("test_tool")
|
|
|
|
assert hints.max_items > 15 # Should be more than default
|
|
assert hints.aggressiveness < 0.7 # Less aggressive
|
|
assert "High retrieval rate" in hints.reason or "less aggressive" in hints.reason.lower()
|
|
|
|
def test_hints_very_high_full_retrieval_skips_compression(self):
|
|
"""Very high full retrieval rate recommends skipping compression."""
|
|
feedback = CompressionFeedback()
|
|
|
|
# 10 compressions
|
|
for _ in range(10):
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
# 9 FULL retrievals (90% retrieval rate, all full)
|
|
for _ in range(9):
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query=None,
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="full",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
hints = feedback.get_compression_hints("test_tool")
|
|
|
|
assert hints.skip_compression is True
|
|
assert "skip compression" in hints.reason.lower()
|
|
|
|
def test_hints_low_retrieval_rate_aggressive(self):
|
|
"""Low retrieval rate means current compression is effective."""
|
|
feedback = CompressionFeedback()
|
|
|
|
# 10 compressions
|
|
for _ in range(10):
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
# Only 1 retrieval (10% - LOW)
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query=None,
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="full",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
hints = feedback.get_compression_hints("test_tool")
|
|
|
|
assert hints.max_items == 15 # Default/aggressive
|
|
assert "effective" in hints.reason.lower() or "Low retrieval" in hints.reason
|
|
|
|
def test_common_queries_tracked(self):
|
|
"""Common search queries are tracked per tool."""
|
|
feedback = CompressionFeedback()
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
queries = ["find errors", "find errors", "status:failed", "error"]
|
|
for q in queries:
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query=q,
|
|
items_retrieved=10,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="search",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
pattern = feedback.get_all_patterns()["test_tool"]
|
|
assert "find errors" in pattern.common_queries
|
|
assert pattern.common_queries["find errors"] == 2
|
|
|
|
def test_queried_fields_extracted(self):
|
|
"""Field names are extracted from queries."""
|
|
feedback = CompressionFeedback()
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
# Query with field:value patterns
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query="status:error id=12345",
|
|
items_retrieved=10,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="search",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
pattern = feedback.get_all_patterns()["test_tool"]
|
|
assert "status" in pattern.queried_fields
|
|
assert "id" in pattern.queried_fields
|
|
|
|
def test_preserve_fields_in_hints(self):
|
|
"""Frequently queried fields appear in hints."""
|
|
feedback = CompressionFeedback()
|
|
|
|
# Multiple compressions
|
|
for _ in range(10):
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
# Multiple queries with same fields
|
|
for _ in range(5):
|
|
event = RetrievalEvent(
|
|
hash="abc123",
|
|
query="status:error code:500",
|
|
items_retrieved=10,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=time.time(),
|
|
retrieval_type="search",
|
|
)
|
|
feedback.record_retrieval(event)
|
|
|
|
hints = feedback.get_compression_hints("test_tool")
|
|
|
|
# Even if retrieval rate triggers hints, preserve_fields should be populated
|
|
assert len(hints.preserve_fields) > 0
|
|
|
|
def test_stats_returns_overview(self):
|
|
"""get_stats returns comprehensive overview."""
|
|
feedback = CompressionFeedback()
|
|
|
|
feedback.record_compression("tool_a", 100, 10)
|
|
feedback.record_compression("tool_b", 200, 20)
|
|
|
|
stats = feedback.get_stats()
|
|
|
|
assert stats["total_compressions"] == 2
|
|
assert stats["tools_tracked"] == 2
|
|
assert "tool_a" in stats["tool_patterns"]
|
|
assert "tool_b" in stats["tool_patterns"]
|
|
|
|
def test_clear_resets_state(self):
|
|
"""clear() removes all learned patterns."""
|
|
feedback = CompressionFeedback()
|
|
feedback.record_compression("test_tool", 100, 10)
|
|
|
|
feedback.clear()
|
|
|
|
assert len(feedback.get_all_patterns()) == 0
|
|
stats = feedback.get_stats()
|
|
assert stats["total_compressions"] == 0
|
|
|
|
|
|
class TestLocalToolPattern:
|
|
"""Test LocalToolPattern dataclass."""
|
|
|
|
def test_retrieval_rate_zero_compressions(self):
|
|
"""Retrieval rate is 0 when no compressions."""
|
|
pattern = LocalToolPattern(tool_name="test")
|
|
assert pattern.retrieval_rate == 0.0
|
|
|
|
def test_full_retrieval_rate_zero_retrievals(self):
|
|
"""Full retrieval rate is 0 when no retrievals."""
|
|
pattern = LocalToolPattern(tool_name="test")
|
|
assert pattern.full_retrieval_rate == 0.0
|
|
|
|
def test_search_rate_calculation(self):
|
|
"""Search rate is calculated correctly."""
|
|
pattern = LocalToolPattern(
|
|
tool_name="test",
|
|
total_retrievals=10,
|
|
full_retrievals=3,
|
|
search_retrievals=7,
|
|
)
|
|
assert pattern.search_rate == 0.7
|
|
|
|
|
|
class TestGlobalFeedback:
|
|
"""Test global feedback singleton."""
|
|
|
|
def test_singleton_returns_same_instance(self):
|
|
"""get_compression_feedback returns same instance."""
|
|
fb1 = get_compression_feedback()
|
|
fb2 = get_compression_feedback()
|
|
assert fb1 is fb2
|
|
|
|
def test_reset_clears_singleton(self):
|
|
"""reset_compression_feedback creates new instance."""
|
|
fb1 = get_compression_feedback()
|
|
fb1.record_compression("test", 100, 10)
|
|
|
|
reset_compression_feedback()
|
|
|
|
fb2 = get_compression_feedback()
|
|
assert len(fb2.get_all_patterns()) == 0
|
|
|
|
|
|
class TestFeedbackIntegrationWithStore:
|
|
"""Test feedback integration with CompressionStore."""
|
|
|
|
def test_store_notifies_feedback_on_retrieval(self):
|
|
"""CompressionStore adds events to pending for feedback processing."""
|
|
store = CompressionStore()
|
|
|
|
# Store content
|
|
hash_key = store.store(
|
|
original='[{"id": 1}, {"id": 2}]',
|
|
compressed='[{"id": 1}]',
|
|
original_item_count=2,
|
|
compressed_item_count=1,
|
|
tool_name="test_tool",
|
|
)
|
|
|
|
# Retrieve (should log event)
|
|
store.retrieve(hash_key)
|
|
|
|
# Process pending events (uses global feedback)
|
|
store.process_pending_feedback()
|
|
|
|
# Now global feedback should have the retrieval
|
|
feedback = get_compression_feedback()
|
|
patterns = feedback.get_all_patterns()
|
|
assert "test_tool" in patterns
|
|
assert patterns["test_tool"].total_retrievals == 1
|