mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
08d81f2e2c
Dashboard was showing wildly incorrect metrics (99.5% savings, 3ms overhead) due to using Anthropic API's non-cached input_tokens instead of optimized_tokens, and dividing overhead by total request count instead of optimized-only count. Key fixes: - Use optimized_tokens (what we sent) for dashboard aggregation, not API's input_tokens which excludes cached portion - Track overhead_count separately from latency_count for correct averages - Add TTFB (time to first byte) measurement, replace full stream latency in UI - Eager-load LLMLingua model at proxy startup (eliminates 5.9s first-request delay) - Simplify CostTracker to token-based accounting with counterfactual cost display - Add two-tier compression cache to ContentRouter (skip set + result cache) - Fix compression pinning to detect both CCR and ReadLifecycle markers - Clamp tokens_saved to max(0, ...) across all provider paths - Add per-transform timing instrumentation to pipeline - Guard against over-aggressive code compression (<5% ratio) - Fix ReadLifecycle partial read supersede logic (_read_covers range check) - Disable CacheAligner and compress_superseded by default - Fix all pre-existing mypy errors (CompressionCache return types) - Fix test mocks to accept **kwargs for cache token parameters
543 lines
19 KiB
Python
543 lines
19 KiB
Python
"""Tests for the config module.
|
|
|
|
Tests all configuration dataclasses, enums, and utility classes:
|
|
- HeadroomMode enum
|
|
- ToolCrusherConfig, CacheAlignerConfig, RollingWindowConfig
|
|
- RelevanceScorerConfig, SmartCrusherConfig
|
|
- HeadroomConfig (main config)
|
|
- Block, WasteSignals, CachePrefixMetrics
|
|
- TransformResult, RequestMetrics
|
|
"""
|
|
|
|
from dataclasses import fields
|
|
from datetime import datetime
|
|
|
|
from headroom.config import (
|
|
Block,
|
|
CacheAlignerConfig,
|
|
CachePrefixMetrics,
|
|
HeadroomConfig,
|
|
HeadroomMode,
|
|
RelevanceScorerConfig,
|
|
RequestMetrics,
|
|
RollingWindowConfig,
|
|
SmartCrusherConfig,
|
|
ToolCrusherConfig,
|
|
TransformResult,
|
|
WasteSignals,
|
|
)
|
|
|
|
|
|
class TestHeadroomMode:
|
|
"""Tests for HeadroomMode enum."""
|
|
|
|
def test_enum_values(self):
|
|
"""All expected enum values exist with correct string values."""
|
|
assert HeadroomMode.AUDIT.value == "audit"
|
|
assert HeadroomMode.OPTIMIZE.value == "optimize"
|
|
assert HeadroomMode.SIMULATE.value == "simulate"
|
|
|
|
def test_string_conversion(self):
|
|
"""HeadroomMode inherits from str for string compatibility."""
|
|
# Enum value access works as string
|
|
assert HeadroomMode.AUDIT.value == "audit"
|
|
assert HeadroomMode.OPTIMIZE.value == "optimize"
|
|
assert HeadroomMode.SIMULATE.value == "simulate"
|
|
# Can compare directly with strings since it inherits from str
|
|
assert HeadroomMode.AUDIT == "audit"
|
|
assert HeadroomMode.OPTIMIZE == "optimize"
|
|
assert HeadroomMode.SIMULATE == "simulate"
|
|
# isinstance check confirms str inheritance
|
|
assert isinstance(HeadroomMode.AUDIT, str)
|
|
|
|
|
|
class TestToolCrusherConfig:
|
|
"""Tests for ToolCrusherConfig dataclass."""
|
|
|
|
def test_default_values(self):
|
|
"""Default values are correctly set."""
|
|
config = ToolCrusherConfig()
|
|
assert config.enabled is False
|
|
assert config.min_tokens_to_crush == 500
|
|
assert config.max_array_items == 10
|
|
assert config.max_string_length == 1000
|
|
assert config.max_depth == 5
|
|
|
|
def test_preserve_keys_default(self):
|
|
"""Default preserve_keys contains expected keys."""
|
|
config = ToolCrusherConfig()
|
|
expected_keys = {"error", "status", "code", "id", "message", "name", "type"}
|
|
assert config.preserve_keys == expected_keys
|
|
# Verify it's a set (mutable default factory)
|
|
assert isinstance(config.preserve_keys, set)
|
|
|
|
def test_tool_profiles_default(self):
|
|
"""Default tool_profiles is an empty dict."""
|
|
config = ToolCrusherConfig()
|
|
assert config.tool_profiles == {}
|
|
assert isinstance(config.tool_profiles, dict)
|
|
|
|
def test_preserve_keys_isolation(self):
|
|
"""Each instance gets its own preserve_keys set."""
|
|
config1 = ToolCrusherConfig()
|
|
config2 = ToolCrusherConfig()
|
|
config1.preserve_keys.add("custom_key")
|
|
assert "custom_key" not in config2.preserve_keys
|
|
|
|
|
|
class TestCacheAlignerConfig:
|
|
"""Tests for CacheAlignerConfig dataclass."""
|
|
|
|
def test_default_values(self):
|
|
"""Default values are correctly set."""
|
|
config = CacheAlignerConfig()
|
|
assert config.enabled is False
|
|
assert config.normalize_whitespace is True
|
|
assert config.collapse_blank_lines is True
|
|
|
|
def test_date_patterns_default(self):
|
|
"""Default date_patterns contains expected regex patterns."""
|
|
config = CacheAlignerConfig()
|
|
assert isinstance(config.date_patterns, list)
|
|
assert len(config.date_patterns) == 4
|
|
# Verify specific patterns exist
|
|
assert r"Current [Dd]ate:?\s*\d{4}-\d{2}-\d{2}" in config.date_patterns
|
|
assert r"Today is \w+,?\s+\w+ \d+" in config.date_patterns
|
|
assert r"Today's date:?\s*\d{4}-\d{2}-\d{2}" in config.date_patterns
|
|
assert r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" in config.date_patterns
|
|
|
|
def test_dynamic_tail_separator_default(self):
|
|
"""Default dynamic_tail_separator has expected value."""
|
|
config = CacheAlignerConfig()
|
|
assert config.dynamic_tail_separator == "\n\n---\n[Dynamic Context]\n"
|
|
|
|
def test_date_patterns_isolation(self):
|
|
"""Each instance gets its own date_patterns list."""
|
|
config1 = CacheAlignerConfig()
|
|
config2 = CacheAlignerConfig()
|
|
config1.date_patterns.append(r"custom pattern")
|
|
assert r"custom pattern" not in config2.date_patterns
|
|
|
|
|
|
class TestRollingWindowConfig:
|
|
"""Tests for RollingWindowConfig dataclass."""
|
|
|
|
def test_default_values(self):
|
|
"""Default values are correctly set."""
|
|
config = RollingWindowConfig()
|
|
assert config.enabled is True
|
|
assert config.keep_last_turns == 2
|
|
|
|
def test_keep_system_default_true(self):
|
|
"""keep_system defaults to True (never drop system prompt)."""
|
|
config = RollingWindowConfig()
|
|
assert config.keep_system is True
|
|
|
|
def test_output_buffer_default(self):
|
|
"""output_buffer_tokens defaults to 4000."""
|
|
config = RollingWindowConfig()
|
|
assert config.output_buffer_tokens == 4000
|
|
|
|
|
|
class TestRelevanceScorerConfig:
|
|
"""Tests for RelevanceScorerConfig dataclass."""
|
|
|
|
def test_default_tier_hybrid(self):
|
|
"""Default tier is hybrid."""
|
|
config = RelevanceScorerConfig()
|
|
assert config.tier == "hybrid"
|
|
|
|
def test_bm25_params(self):
|
|
"""BM25 parameters have expected defaults."""
|
|
config = RelevanceScorerConfig()
|
|
assert config.bm25_k1 == 1.5
|
|
assert config.bm25_b == 0.75
|
|
|
|
def test_embedding_params(self):
|
|
"""Embedding parameters have expected defaults."""
|
|
config = RelevanceScorerConfig()
|
|
assert config.embedding_model == "all-MiniLM-L6-v2"
|
|
assert config.hybrid_alpha == 0.5
|
|
assert config.adaptive_alpha is True
|
|
|
|
def test_relevance_threshold_default(self):
|
|
"""Relevance threshold defaults to 0.25."""
|
|
config = RelevanceScorerConfig()
|
|
assert config.relevance_threshold == 0.25
|
|
|
|
|
|
class TestSmartCrusherConfig:
|
|
"""Tests for SmartCrusherConfig dataclass."""
|
|
|
|
def test_default_values(self):
|
|
"""Default values are correctly set."""
|
|
config = SmartCrusherConfig()
|
|
assert config.min_items_to_analyze == 5
|
|
assert config.min_tokens_to_crush == 200
|
|
assert config.variance_threshold == 2.0
|
|
assert config.uniqueness_threshold == 0.1
|
|
assert config.similarity_threshold == 0.8
|
|
assert config.max_items_after_crush == 15
|
|
assert config.preserve_change_points is True
|
|
assert config.factor_out_constants is False
|
|
assert config.include_summaries is False
|
|
|
|
def test_enabled_by_default(self):
|
|
"""SmartCrusher is enabled by default."""
|
|
config = SmartCrusherConfig()
|
|
assert config.enabled is True
|
|
|
|
def test_relevance_field_default(self):
|
|
"""Relevance field defaults to RelevanceScorerConfig instance."""
|
|
config = SmartCrusherConfig()
|
|
assert isinstance(config.relevance, RelevanceScorerConfig)
|
|
assert config.relevance.tier == "hybrid"
|
|
|
|
def test_relevance_isolation(self):
|
|
"""Each instance gets its own RelevanceScorerConfig."""
|
|
config1 = SmartCrusherConfig()
|
|
config2 = SmartCrusherConfig()
|
|
config1.relevance.tier = "bm25"
|
|
assert config2.relevance.tier == "hybrid"
|
|
|
|
|
|
class TestHeadroomConfig:
|
|
"""Tests for HeadroomConfig main configuration class."""
|
|
|
|
def test_default_values(self):
|
|
"""Default values are correctly set."""
|
|
config = HeadroomConfig()
|
|
assert config.store_url == "sqlite:///headroom.db"
|
|
assert config.default_mode == HeadroomMode.AUDIT
|
|
assert config.generate_diff_artifact is False
|
|
# Nested configs exist
|
|
assert isinstance(config.tool_crusher, ToolCrusherConfig)
|
|
assert isinstance(config.smart_crusher, SmartCrusherConfig)
|
|
assert isinstance(config.cache_aligner, CacheAlignerConfig)
|
|
assert isinstance(config.rolling_window, RollingWindowConfig)
|
|
|
|
def test_get_context_limit_direct_match(self):
|
|
"""get_context_limit returns limit for exact model match."""
|
|
config = HeadroomConfig(model_context_limits={"gpt-4o": 128000, "claude-3-opus": 200000})
|
|
assert config.get_context_limit("gpt-4o") == 128000
|
|
assert config.get_context_limit("claude-3-opus") == 200000
|
|
|
|
def test_get_context_limit_prefix_match(self):
|
|
"""get_context_limit returns limit for prefix match."""
|
|
config = HeadroomConfig(model_context_limits={"gpt-4": 128000, "claude-3": 200000})
|
|
# Prefix matches
|
|
assert config.get_context_limit("gpt-4-turbo") == 128000
|
|
assert config.get_context_limit("gpt-4o") == 128000
|
|
assert config.get_context_limit("claude-3-opus") == 200000
|
|
assert config.get_context_limit("claude-3-sonnet") == 200000
|
|
|
|
def test_get_context_limit_not_found(self):
|
|
"""get_context_limit returns None for unknown model."""
|
|
config = HeadroomConfig(model_context_limits={"gpt-4": 128000})
|
|
assert config.get_context_limit("unknown-model") is None
|
|
assert config.get_context_limit("llama-2") is None
|
|
|
|
def test_model_context_limits_isolation(self):
|
|
"""Each instance gets its own model_context_limits dict."""
|
|
config1 = HeadroomConfig()
|
|
config2 = HeadroomConfig()
|
|
config1.model_context_limits["custom-model"] = 50000
|
|
assert "custom-model" not in config2.model_context_limits
|
|
|
|
|
|
class TestBlock:
|
|
"""Tests for Block dataclass."""
|
|
|
|
def test_block_creation(self):
|
|
"""Block can be created with required fields."""
|
|
block = Block(
|
|
kind="user",
|
|
text="Hello, world!",
|
|
tokens_est=5,
|
|
content_hash="abc123",
|
|
source_index=0,
|
|
)
|
|
assert block.kind == "user"
|
|
assert block.text == "Hello, world!"
|
|
assert block.tokens_est == 5
|
|
assert block.content_hash == "abc123"
|
|
assert block.source_index == 0
|
|
assert block.flags == {}
|
|
|
|
def test_block_kinds(self):
|
|
"""Block accepts all valid kind values."""
|
|
valid_kinds = ["system", "user", "assistant", "tool_call", "tool_result", "rag", "unknown"]
|
|
for kind in valid_kinds:
|
|
block = Block(
|
|
kind=kind,
|
|
text="test",
|
|
tokens_est=1,
|
|
content_hash="hash",
|
|
source_index=0,
|
|
)
|
|
assert block.kind == kind
|
|
|
|
def test_block_flags_default_factory(self):
|
|
"""Each block gets its own flags dict."""
|
|
block1 = Block(kind="user", text="a", tokens_est=1, content_hash="h1", source_index=0)
|
|
block2 = Block(kind="user", text="b", tokens_est=1, content_hash="h2", source_index=1)
|
|
block1.flags["custom"] = True
|
|
assert "custom" not in block2.flags
|
|
|
|
|
|
class TestWasteSignals:
|
|
"""Tests for WasteSignals dataclass."""
|
|
|
|
def test_total_calculation(self):
|
|
"""total() correctly sums all waste token fields."""
|
|
signals = WasteSignals(
|
|
json_bloat_tokens=100,
|
|
html_noise_tokens=50,
|
|
base64_tokens=200,
|
|
whitespace_tokens=25,
|
|
dynamic_date_tokens=10,
|
|
repetition_tokens=15,
|
|
)
|
|
assert signals.total() == 400
|
|
|
|
def test_total_with_defaults(self):
|
|
"""total() returns 0 when all fields are default."""
|
|
signals = WasteSignals()
|
|
assert signals.total() == 0
|
|
|
|
def test_to_dict(self):
|
|
"""to_dict() returns correct dictionary representation."""
|
|
signals = WasteSignals(
|
|
json_bloat_tokens=100,
|
|
html_noise_tokens=50,
|
|
base64_tokens=200,
|
|
whitespace_tokens=25,
|
|
dynamic_date_tokens=10,
|
|
repetition_tokens=15,
|
|
)
|
|
expected = {
|
|
"json_bloat": 100,
|
|
"html_noise": 50,
|
|
"base64": 200,
|
|
"whitespace": 25,
|
|
"dynamic_date": 10,
|
|
"repetition": 15,
|
|
}
|
|
assert signals.to_dict() == expected
|
|
|
|
def test_to_dict_defaults(self):
|
|
"""to_dict() returns zeroes for default values."""
|
|
signals = WasteSignals()
|
|
result = signals.to_dict()
|
|
assert all(v == 0 for v in result.values())
|
|
assert len(result) == 6
|
|
|
|
|
|
class TestCachePrefixMetrics:
|
|
"""Tests for CachePrefixMetrics dataclass."""
|
|
|
|
def test_dataclass_fields(self):
|
|
"""CachePrefixMetrics has all expected fields."""
|
|
field_names = {f.name for f in fields(CachePrefixMetrics)}
|
|
expected_fields = {
|
|
"stable_prefix_bytes",
|
|
"stable_prefix_tokens_est",
|
|
"stable_prefix_hash",
|
|
"prefix_changed",
|
|
"previous_hash",
|
|
}
|
|
assert field_names == expected_fields
|
|
|
|
def test_creation(self):
|
|
"""CachePrefixMetrics can be created with required fields."""
|
|
metrics = CachePrefixMetrics(
|
|
stable_prefix_bytes=1024,
|
|
stable_prefix_tokens_est=256,
|
|
stable_prefix_hash="abc123def456",
|
|
prefix_changed=False,
|
|
)
|
|
assert metrics.stable_prefix_bytes == 1024
|
|
assert metrics.stable_prefix_tokens_est == 256
|
|
assert metrics.stable_prefix_hash == "abc123def456"
|
|
assert metrics.prefix_changed is False
|
|
assert metrics.previous_hash is None
|
|
|
|
def test_previous_hash_optional(self):
|
|
"""previous_hash defaults to None."""
|
|
metrics = CachePrefixMetrics(
|
|
stable_prefix_bytes=512,
|
|
stable_prefix_tokens_est=128,
|
|
stable_prefix_hash="hash123",
|
|
prefix_changed=True,
|
|
previous_hash="oldhash",
|
|
)
|
|
assert metrics.previous_hash == "oldhash"
|
|
|
|
|
|
class TestTransformResult:
|
|
"""Tests for TransformResult dataclass."""
|
|
|
|
def test_dataclass_fields(self):
|
|
"""TransformResult has all expected fields."""
|
|
field_names = {f.name for f in fields(TransformResult)}
|
|
expected_fields = {
|
|
"messages",
|
|
"tokens_before",
|
|
"tokens_after",
|
|
"transforms_applied",
|
|
"markers_inserted",
|
|
"warnings",
|
|
"diff_artifact",
|
|
"cache_metrics",
|
|
"timing",
|
|
"waste_signals",
|
|
}
|
|
assert field_names == expected_fields
|
|
|
|
def test_default_empty_lists(self):
|
|
"""Default factory produces empty lists for optional fields."""
|
|
result = TransformResult(
|
|
messages=[{"role": "user", "content": "test"}],
|
|
tokens_before=100,
|
|
tokens_after=80,
|
|
transforms_applied=["CacheAligner"],
|
|
)
|
|
assert result.markers_inserted == []
|
|
assert result.warnings == []
|
|
assert result.diff_artifact is None
|
|
assert result.cache_metrics is None
|
|
|
|
def test_list_isolation(self):
|
|
"""Each instance gets its own lists."""
|
|
result1 = TransformResult(
|
|
messages=[],
|
|
tokens_before=100,
|
|
tokens_after=80,
|
|
transforms_applied=["Transform1"],
|
|
)
|
|
result2 = TransformResult(
|
|
messages=[],
|
|
tokens_before=100,
|
|
tokens_after=80,
|
|
transforms_applied=["Transform2"],
|
|
)
|
|
result1.markers_inserted.append("marker")
|
|
result1.warnings.append("warning")
|
|
assert result2.markers_inserted == []
|
|
assert result2.warnings == []
|
|
|
|
|
|
class TestRequestMetrics:
|
|
"""Tests for RequestMetrics dataclass."""
|
|
|
|
def test_dataclass_fields(self):
|
|
"""RequestMetrics has all expected fields."""
|
|
field_names = {f.name for f in fields(RequestMetrics)}
|
|
expected_fields = {
|
|
"request_id",
|
|
"timestamp",
|
|
"model",
|
|
"stream",
|
|
"mode",
|
|
"tokens_input_before",
|
|
"tokens_input_after",
|
|
"tokens_output",
|
|
"block_breakdown",
|
|
"waste_signals",
|
|
"stable_prefix_hash",
|
|
"cache_alignment_score",
|
|
"cached_tokens",
|
|
# Cache optimizer metrics (provider-specific)
|
|
"cache_optimizer_used",
|
|
"cache_optimizer_strategy",
|
|
"cacheable_tokens",
|
|
"breakpoints_inserted",
|
|
"estimated_cache_hit",
|
|
"estimated_savings_percent",
|
|
"semantic_cache_hit",
|
|
# Transform details
|
|
"transforms_applied",
|
|
"tool_units_dropped",
|
|
"turns_dropped",
|
|
"messages_hash",
|
|
"error",
|
|
}
|
|
assert field_names == expected_fields
|
|
|
|
def test_default_values(self):
|
|
"""Default values are correctly set for optional fields."""
|
|
metrics = RequestMetrics(
|
|
request_id="test-123",
|
|
timestamp=datetime(2025, 1, 6),
|
|
model="gpt-4o",
|
|
stream=False,
|
|
mode="audit",
|
|
tokens_input_before=1000,
|
|
tokens_input_after=800,
|
|
)
|
|
assert metrics.tokens_output is None
|
|
assert metrics.block_breakdown == {}
|
|
assert metrics.waste_signals == {}
|
|
assert metrics.stable_prefix_hash == ""
|
|
assert metrics.cache_alignment_score == 0.0
|
|
assert metrics.cached_tokens is None
|
|
assert metrics.transforms_applied == []
|
|
assert metrics.tool_units_dropped == 0
|
|
assert metrics.turns_dropped == 0
|
|
assert metrics.messages_hash == ""
|
|
assert metrics.error is None
|
|
|
|
def test_full_creation(self):
|
|
"""RequestMetrics can be created with all fields."""
|
|
metrics = RequestMetrics(
|
|
request_id="req-456",
|
|
timestamp=datetime(2025, 1, 6, 12, 30),
|
|
model="claude-3-opus",
|
|
stream=True,
|
|
mode="optimize",
|
|
tokens_input_before=2000,
|
|
tokens_input_after=1500,
|
|
tokens_output=500,
|
|
block_breakdown={"system": 200, "user": 800},
|
|
waste_signals={"json_bloat": 100},
|
|
stable_prefix_hash="hash123",
|
|
cache_alignment_score=95.5,
|
|
cached_tokens=200,
|
|
transforms_applied=["CacheAligner", "SmartCrusher"],
|
|
tool_units_dropped=2,
|
|
turns_dropped=1,
|
|
messages_hash="msghash",
|
|
error=None,
|
|
)
|
|
assert metrics.request_id == "req-456"
|
|
assert metrics.model == "claude-3-opus"
|
|
assert metrics.stream is True
|
|
assert metrics.tokens_output == 500
|
|
assert metrics.cache_alignment_score == 95.5
|
|
|
|
def test_dict_isolation(self):
|
|
"""Each instance gets its own dicts and lists."""
|
|
metrics1 = RequestMetrics(
|
|
request_id="1",
|
|
timestamp=datetime.now(),
|
|
model="m",
|
|
stream=False,
|
|
mode="audit",
|
|
tokens_input_before=100,
|
|
tokens_input_after=100,
|
|
)
|
|
metrics2 = RequestMetrics(
|
|
request_id="2",
|
|
timestamp=datetime.now(),
|
|
model="m",
|
|
stream=False,
|
|
mode="audit",
|
|
tokens_input_before=100,
|
|
tokens_input_after=100,
|
|
)
|
|
metrics1.block_breakdown["system"] = 50
|
|
metrics1.waste_signals["json_bloat"] = 25
|
|
metrics1.transforms_applied.append("Test")
|
|
assert metrics2.block_breakdown == {}
|
|
assert metrics2.waste_signals == {}
|
|
assert metrics2.transforms_applied == []
|