mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
c765c53bf8
Stage 3c.1b step 2 + cleanup. The python `SmartCrusher` (3669 lines) is replaced by a thin pyo3-backed shim (~290 lines) that delegates every byte to `headroom._core.SmartCrusher` (built from `crates/headroom-py`, landed in the previous commit). There is no python implementation and no env-var fallback — the wheel is a hard import. Why now: parity was already proven across 17 fixtures + the python- side bridge test (1+17 in `test_smart_crusher_rust_parity.py`). Keeping a shadow python impl behind a flag is a permanent maintenance cost with no operational benefit. Stage 3c.1b deletes ~3380 lines of python parser/scorer/analyzer/orchestrator code; the rust crate has its own coverage (388 unit tests + property tests in headroom-core). Surface preserved (drop-in for every production caller): - `headroom.transforms.smart_crusher.SmartCrusher` — same class name, same `__init__(config, relevance_config, scorer, ccr_config)` signature (the latter three are accepted for source-compat and silently dropped — rust port keeps those subsystems disabled in Stage 3c.1, they re-attach in Stage 3c.2). - `SmartCrusherConfig` and `CrushResult` dataclasses kept as python dataclasses (callers use `asdict()` / dataclass matching on them). - `crush(content, query, bias)`, `_smart_crush_content(content, ...)`, `apply(messages, tokenizer, **kwargs)`, and `_extract_context_from_messages(messages)` all preserved. - `smart_crush_tool_output(content, config, ccr_config)` thin wrapper. The transform-protocol `apply()` orchestration stays python (message walking, digest-marker insertion, token counting); only the per- message compression call delegates to rust. Removed: - Python parser / planner / scorer / analyzer / classifier (~3380 lines). - Internal helpers `_classify_array`, `_detect_sequential_pattern`, `_detect_rare_status_values`, `_detect_items_by_learned_semantics`, `_percentile_linear`, `_compute_k_split`, `_crush_number_array`, `_process_value`, etc. — rust crate has parallel coverage. - `SmartAnalyzer`, `ArrayType`, `CompressionStrategy`, `extract_query_anchors` — internals; not used by any production caller (only tests probed them). Tests deleted (probed deleted internals — same precedent as Stage 3b): - `tests/test_transforms/test_smart_crusher.py` (40 tests) - `tests/test_transforms/test_universal_json_crush.py` (45) - `tests/test_transforms/test_anchor_selector.py` (49) - `tests/test_toin_field_learning.py` (21) - `tests/test_crushability.py` (20) Tests trimmed (removed methods/classes that probe deferred subsystems — scorer injection, CCR marker injection, TOIN feedback recording — all of which re-attach in Stage 3c.2): - `tests/test_transforms/test_smart_crusher_bugs.py`: TestNumberArraySchemaPreservation, TestStage3c1BugFixes. - `tests/test_relevance.py`: 2 scorer-injection tests. - `tests/test_ccr.py`: TestSmartCrusherCCRIntegration class + test_custom_marker_template. - `tests/test_toin_integration.py`: TestTOINIntegration + TestStoreToTOINHash classes. - `tests/test_critical_fixes.py`: TestSmartCrusherTOINIntegration + test_full_feedback_loop. - `tests/test_acceptance.py::TestQueryAnchorExtraction`: dropped the `extract_query_anchors` probe; kept the end-to-end "Alice preserved" assertion. Bug fixes from Stage 3c.1 (#1 percentile linear interp, #2 zero- padded sequential, #3 rare-status pareto, #4 k-split overshoot) are pinned by the rust crate and the parity fixtures (`tests/parity/fixtures/smart_crusher/`). Tests: - 517 passed in the smart_crusher-adjacent file set (test_transforms/, test_relevance*, test_ccr, test_toin_integration, test_quality_retention, test_acceptance, test_critical_fixes). - 18 in `test_smart_crusher_rust_parity.py` (1 sanity + 17 fixtures). - 388 rust unit tests still green. One stale-error-message regex in `test_relevance_extra.py` updated from "requires sentence-transformers" → "requires fastembed".
181 lines
6.7 KiB
Python
181 lines
6.7 KiB
Python
from __future__ import annotations
|
||
|
||
import builtins
|
||
from dataclasses import dataclass
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
|
||
import headroom.relevance as relevance_mod
|
||
from headroom.relevance import (
|
||
BM25Scorer,
|
||
EmbeddingScorer,
|
||
HybridScorer,
|
||
create_scorer,
|
||
embedding,
|
||
hybrid,
|
||
)
|
||
from headroom.relevance.base import RelevanceScore, RelevanceScorer, default_batch_score
|
||
|
||
|
||
@dataclass
|
||
class DummyRelevanceScorer(RelevanceScorer):
|
||
def score(self, item: str, context: str) -> RelevanceScore:
|
||
return RelevanceScore(score=0.4, reason=f"{item}:{context}")
|
||
|
||
def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:
|
||
return [RelevanceScore(score=0.2, reason=context) for _ in items]
|
||
|
||
|
||
def test_base_default_batch_and_abstract_methods() -> None:
|
||
scorer = DummyRelevanceScorer()
|
||
batch = default_batch_score(scorer, ["a", "b"], "ctx")
|
||
assert [item.reason for item in batch] == ["a:ctx", "b:ctx"]
|
||
|
||
assert RelevanceScorer.score(scorer, "a", "ctx") is None
|
||
assert RelevanceScorer.score_batch(scorer, ["a"], "ctx") is None
|
||
assert RelevanceScorer.is_available() is True
|
||
|
||
|
||
def test_create_scorer_embedding_unavailable_branch(monkeypatch) -> None:
|
||
monkeypatch.setattr(
|
||
relevance_mod.EmbeddingScorer, "is_available", classmethod(lambda cls: False)
|
||
)
|
||
with pytest.raises(RuntimeError, match="sentence-transformers"):
|
||
create_scorer("embedding")
|
||
|
||
|
||
def test_bm25_internal_paths_and_non_normalized_mode() -> None:
|
||
scorer = BM25Scorer(normalize_score=False)
|
||
assert scorer._tokenize("") == []
|
||
assert scorer._compute_idf("x", doc_count=1, doc_freq=0) == 0.0
|
||
assert scorer._compute_idf("x", doc_count=1, doc_freq=1) > 0
|
||
assert scorer._bm25_score([], ["a"]) == (0.0, [])
|
||
assert scorer._bm25_score(["a"], []) == (0.0, [])
|
||
|
||
no_match = scorer.score("hello world", "missing")
|
||
assert no_match.reason == "BM25: no term matches"
|
||
|
||
one_match = scorer.score("find alice", "alice")
|
||
assert one_match.reason == "BM25: matched 'alice'"
|
||
assert one_match.score > 0
|
||
|
||
many_match = scorer.score("alpha beta gamma delta", "alpha beta gamma delta")
|
||
assert many_match.reason.startswith("BM25: matched 4 terms")
|
||
|
||
batch = scorer.score_batch(["alpha", "alpha beta"], "alpha beta")
|
||
assert [item.reason for item in batch] == ["BM25: 1 terms", "BM25: 2 terms"]
|
||
|
||
|
||
def test_embedding_numpy_and_model_error_paths(monkeypatch) -> None:
|
||
embedding._numpy = None
|
||
real_import = builtins.__import__
|
||
|
||
def fake_import(name, globals=None, locals=None, fromlist=(), level=0):
|
||
if name == "numpy":
|
||
raise ImportError("missing")
|
||
return real_import(name, globals, locals, fromlist, level)
|
||
|
||
monkeypatch.setattr(builtins, "__import__", fake_import)
|
||
with pytest.raises(ImportError, match="numpy is required"):
|
||
embedding._get_numpy()
|
||
|
||
monkeypatch.setattr(builtins, "__import__", real_import)
|
||
fake_np = SimpleNamespace(
|
||
linalg=SimpleNamespace(norm=lambda value: 0 if value == [0, 0] else 1),
|
||
dot=lambda a, b: -1,
|
||
)
|
||
monkeypatch.setattr(embedding, "_numpy", fake_np)
|
||
assert embedding._cosine_similarity([0, 0], [1, 0]) == 0.0
|
||
assert embedding._cosine_similarity([1, 0], [0, 1]) == 0.0
|
||
|
||
monkeypatch.setattr(EmbeddingScorer, "is_available", classmethod(lambda cls: False))
|
||
with pytest.raises(RuntimeError, match="requires fastembed"):
|
||
EmbeddingScorer()._get_model()
|
||
|
||
|
||
def test_embedding_score_empty_and_batch_shortcuts() -> None:
|
||
scorer = EmbeddingScorer()
|
||
assert scorer.score("", "ctx").reason == "Embedding: empty input"
|
||
assert scorer.score("item", "").reason == "Embedding: empty input"
|
||
assert scorer.score_batch([], "ctx") == []
|
||
assert scorer.score_batch(["item"], "")[0].reason == "Embedding: empty context"
|
||
|
||
|
||
def test_embedding_score_and_batch_with_fake_model(monkeypatch) -> None:
|
||
scorer = EmbeddingScorer()
|
||
monkeypatch.setattr(
|
||
scorer,
|
||
"_encode",
|
||
lambda texts: (
|
||
[[1.0, 0.0], [0.5, 0.5]] if len(texts) == 2 else [[1.0, 0.0], [0.0, 1.0], [1.0, 0.0]]
|
||
),
|
||
)
|
||
monkeypatch.setattr(
|
||
embedding, "_cosine_similarity", lambda a, b: 0.75 if a == [1.0, 0.0] else 0.25
|
||
)
|
||
|
||
single = scorer.score("item", "ctx")
|
||
assert single.score == 0.75
|
||
assert single.reason == "Embedding: semantic similarity 0.75"
|
||
|
||
batch = scorer.score_batch(["first", "second"], "ctx")
|
||
assert [item.score for item in batch] == [0.75, 0.25]
|
||
assert [item.reason for item in batch] == ["Embedding: 0.75", "Embedding: 0.25"]
|
||
|
||
|
||
def test_hybrid_constructor_alpha_variants_and_single_score_paths(monkeypatch) -> None:
|
||
bm25_result = RelevanceScore(score=0.1, reason="bm25", matched_terms=["term"])
|
||
emb_result = RelevanceScore(score=0.9, reason="emb", matched_terms=[])
|
||
|
||
class FakeBM25:
|
||
def score(self, item: str, context: str) -> RelevanceScore:
|
||
return bm25_result
|
||
|
||
def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:
|
||
return [bm25_result for _ in items]
|
||
|
||
class FakeEmbedding:
|
||
def score(self, item: str, context: str) -> RelevanceScore:
|
||
return emb_result
|
||
|
||
def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:
|
||
return [emb_result for _ in items]
|
||
|
||
scorer = HybridScorer(
|
||
alpha=0.4, adaptive=True, bm25_scorer=FakeBM25(), embedding_scorer=FakeEmbedding()
|
||
)
|
||
assert scorer.has_embedding_support() is True
|
||
assert scorer._compute_alpha("find id 1234") == 0.65
|
||
assert scorer._compute_alpha("find host api.example.com") == 0.6
|
||
assert scorer._compute_alpha("find email test@example.com") == 0.6
|
||
|
||
single = scorer.score("item", "show me errors")
|
||
assert single.score == pytest.approx(0.58)
|
||
assert "Hybrid (α=0.40): BM25=0.10, Semantic=0.90" == single.reason
|
||
|
||
batch = scorer.score_batch(["a", "b"], "show me errors")
|
||
assert len(batch) == 2
|
||
assert batch[0].reason == "Hybrid (α=0.40): BM25=0.10, Emb=0.90"
|
||
|
||
|
||
def test_hybrid_fallback_and_empty_batch(monkeypatch) -> None:
|
||
scorer = HybridScorer(bm25_scorer=BM25Scorer())
|
||
scorer._embedding_available = False
|
||
scorer.embedding = None
|
||
|
||
empty = scorer.score_batch([], "ctx")
|
||
assert empty == []
|
||
|
||
boosted = scorer.score('{"id":"123","name":"alice"}', "alice")
|
||
assert boosted.score >= 0.3
|
||
assert "BM25 only, boosted" in boosted.reason
|
||
|
||
boosted_batch = scorer.score_batch(['{"id":"123"}', '{"id":"456"}'], "123 456")
|
||
assert all("BM25 only, boosted" in item.reason for item in boosted_batch)
|
||
|
||
|
||
def test_hybrid_auto_fallback_when_embeddings_unavailable(monkeypatch) -> None:
|
||
monkeypatch.setattr(hybrid.EmbeddingScorer, "is_available", classmethod(lambda cls: False))
|
||
scorer = HybridScorer()
|
||
assert scorer.has_embedding_support() is False
|