mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
chore(transforms): retire dead text_compressor module (Phase 3e.3)
`headroom/transforms/text_compressor.py` was a regex-line-sampling fallback that nothing in the runtime called. ContentRouter routes `CompressionStrategy.TEXT` straight to the Kompress ML compressor at `content_router.py:1046` — the comment there literally says 'Prefer Kompress ML compressor for text'. The Python file was orphaned but still imported by its own test class, making it look live in the 3e queue. Drops the 3e.3 port from the queue: there's nothing to port. # Removed * `headroom/transforms/text_compressor.py` (255 LOC, unused) * `tests/test_text_compressors.py::TestTextCompressor` (3 tests) * `text_compressor` mention in `error_detection.py` shim docstring * `text_compressor` mention in `test_signals_keyword_parity.py` docstring * `TextCompressor` mention in `bench_latency.py` scenario comment # Kept (defensive) The legacy marker regex in `ccr/tool_injection.py:213` stays — it parses an even older TextCompressor output format (pre-2026), is purely defensive, and removal buys nothing. Test references to that format in `test_ccr_tool_injection.py` document the regex contract and stay too. # Test plan * `make ci-precheck` clean * `tests/test_text_compressors.py` 19 passes (was 22, dropped 3)
This commit is contained in:
chopratejas
parent
2228d1b019
commit
0161cdb386
5 changed files with 9 additions and 306 deletions
|
|
@ -28,7 +28,7 @@ Usage:
|
|||
Scenarios:
|
||||
json - JSON arrays via SmartCrusher (100-5K items)
|
||||
code - Python source via CodeCompressor (50-1000 lines)
|
||||
text - Plain text/RAG via TextCompressor (1K-50K tokens)
|
||||
text - Plain text/RAG via Kompress fallback (1K-50K tokens)
|
||||
logs - Structured logs via LogCompressor (100-5K entries)
|
||||
agentic - Multi-turn agent conversations (10-100 turns)
|
||||
rag - RAG conversations with large context (5K-50K tokens)
|
||||
|
|
@ -536,7 +536,7 @@ def generate_scenarios(content_types: list[str] | None = None) -> list[Scenario]
|
|||
)
|
||||
)
|
||||
|
||||
# --- Plain text (TextCompressor path) ---
|
||||
# --- Plain text (Kompress fallback path) ---
|
||||
if "text" in types:
|
||||
for tokens, label in [
|
||||
(1_000, "1K tokens"),
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ Phase 3e.1 ported the keyword data + scoring logic to
|
|||
truth.
|
||||
2. Re-exports the legacy ``frozenset`` and compiled-regex names
|
||||
(``ERROR_KEYWORDS``, ``ERROR_PATTERN``, ``PRIORITY_PATTERNS_TEXT``,
|
||||
…) so the existing callers in ``text_compressor``,
|
||||
``search_compressor``, ``diff_compressor``, and
|
||||
``intelligent_context`` keep working without same-PR refactors.
|
||||
…) so the existing callers in ``search_compressor``,
|
||||
``diff_compressor``, and ``intelligent_context`` keep working
|
||||
without same-PR refactors.
|
||||
3. Delegates ``content_has_error_indicators`` to the Rust
|
||||
aho-corasick automaton.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,255 +0,0 @@
|
|||
"""Generic text compressor for plain text content.
|
||||
|
||||
This module provides a fallback compressor for plain text that doesn't match
|
||||
any specialized format (search results, logs, code, diffs). Uses line-based
|
||||
sampling with anchor preservation.
|
||||
|
||||
Compression Strategy:
|
||||
1. Identify anchor lines (contain context keywords)
|
||||
2. Keep first N and last M lines
|
||||
3. Sample from middle based on line importance
|
||||
4. Add summary of omitted content
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextCompressorConfig:
|
||||
"""Configuration for text compression."""
|
||||
|
||||
# Line limits
|
||||
keep_first_lines: int = 10
|
||||
keep_last_lines: int = 10
|
||||
max_total_lines: int = 50
|
||||
|
||||
# Sampling
|
||||
sample_every_n_lines: int = 10
|
||||
|
||||
# Anchor detection
|
||||
anchor_keywords: list[str] = field(default_factory=list)
|
||||
boost_pattern_lines: bool = True
|
||||
|
||||
# CCR integration
|
||||
enable_ccr: bool = True
|
||||
min_lines_for_ccr: int = 100
|
||||
|
||||
|
||||
class TextCompressor:
|
||||
"""Compresses generic plain text.
|
||||
|
||||
Example:
|
||||
>>> compressor = TextCompressor()
|
||||
>>> result = compressor.compress(large_text, context="find errors")
|
||||
>>> print(result.compressed)
|
||||
"""
|
||||
|
||||
# Patterns that indicate important lines (centralized in error_detection module)
|
||||
from headroom.transforms.error_detection import PRIORITY_PATTERNS_TEXT
|
||||
|
||||
_IMPORTANT_PATTERNS = PRIORITY_PATTERNS_TEXT
|
||||
|
||||
def __init__(self, config: TextCompressorConfig | None = None):
|
||||
"""Initialize text compressor.
|
||||
|
||||
Args:
|
||||
config: Compression configuration.
|
||||
"""
|
||||
self.config = config or TextCompressorConfig()
|
||||
|
||||
def compress(self, content: str, context: str = "") -> TextCompressionResult:
|
||||
"""Compress text content.
|
||||
|
||||
Args:
|
||||
content: Raw text content.
|
||||
context: User query context for anchor detection.
|
||||
|
||||
Returns:
|
||||
TextCompressionResult with compressed output.
|
||||
"""
|
||||
lines = content.split("\n")
|
||||
|
||||
if len(lines) <= self.config.max_total_lines:
|
||||
return TextCompressionResult(
|
||||
compressed=content,
|
||||
original=content,
|
||||
original_line_count=len(lines),
|
||||
compressed_line_count=len(lines),
|
||||
compression_ratio=1.0,
|
||||
)
|
||||
|
||||
# Score lines by importance
|
||||
scored_lines = self._score_lines(lines, context)
|
||||
|
||||
# Select lines
|
||||
selected = self._select_lines(scored_lines, lines)
|
||||
|
||||
# Format output
|
||||
compressed = self._format_output(selected, len(lines))
|
||||
|
||||
ratio = len(compressed) / max(len(content), 1)
|
||||
|
||||
# Store in CCR if significant compression
|
||||
cache_key = None
|
||||
if self.config.enable_ccr and len(lines) >= self.config.min_lines_for_ccr and ratio < 0.7:
|
||||
cache_key = self._store_in_ccr(content, compressed, len(lines))
|
||||
if cache_key:
|
||||
# Use consistent CCR marker format for CCRToolInjector detection
|
||||
compressed += f"\n[{len(lines)} lines compressed to {len(selected)}. Retrieve more: hash={cache_key}]"
|
||||
|
||||
return TextCompressionResult(
|
||||
compressed=compressed,
|
||||
original=content,
|
||||
original_line_count=len(lines),
|
||||
compressed_line_count=len(selected),
|
||||
compression_ratio=ratio,
|
||||
cache_key=cache_key,
|
||||
)
|
||||
|
||||
def _score_lines(self, lines: list[str], context: str) -> list[tuple[int, str, float]]:
|
||||
"""Score lines by importance."""
|
||||
context_lower = context.lower()
|
||||
context_words = set(context_lower.split()) if context else set()
|
||||
anchor_keywords = {k.lower() for k in self.config.anchor_keywords}
|
||||
|
||||
scored: list[tuple[int, str, float]] = []
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
score = 0.0
|
||||
line_lower = line.lower()
|
||||
|
||||
# Boost if contains context words
|
||||
for word in context_words:
|
||||
if len(word) > 2 and word in line_lower:
|
||||
score += 0.3
|
||||
|
||||
# Boost if contains anchor keywords
|
||||
for keyword in anchor_keywords:
|
||||
if keyword in line_lower:
|
||||
score += 0.4
|
||||
|
||||
# Boost if matches important patterns
|
||||
if self.config.boost_pattern_lines:
|
||||
for pattern in self._IMPORTANT_PATTERNS:
|
||||
if pattern.search(line):
|
||||
score += 0.2
|
||||
break
|
||||
|
||||
# Small boost for non-empty lines
|
||||
if line.strip():
|
||||
score += 0.1
|
||||
|
||||
scored.append((i, line, min(1.0, score)))
|
||||
|
||||
return scored
|
||||
|
||||
def _select_lines(
|
||||
self, scored_lines: list[tuple[int, str, float]], original_lines: list[str]
|
||||
) -> list[tuple[int, str]]:
|
||||
"""Select lines to keep."""
|
||||
total = len(scored_lines)
|
||||
selected_indices: set[int] = set()
|
||||
|
||||
# Always keep first N lines
|
||||
for i in range(min(self.config.keep_first_lines, total)):
|
||||
selected_indices.add(i)
|
||||
|
||||
# Always keep last M lines
|
||||
for i in range(max(0, total - self.config.keep_last_lines), total):
|
||||
selected_indices.add(i)
|
||||
|
||||
# Add high-scoring lines
|
||||
high_score_lines = [
|
||||
(idx, line, score)
|
||||
for idx, line, score in scored_lines
|
||||
if score >= 0.3 and idx not in selected_indices
|
||||
]
|
||||
high_score_lines.sort(key=lambda x: x[2], reverse=True)
|
||||
|
||||
remaining_slots = self.config.max_total_lines - len(selected_indices)
|
||||
for idx, _line, _score in high_score_lines[:remaining_slots]:
|
||||
selected_indices.add(idx)
|
||||
remaining_slots -= 1
|
||||
if remaining_slots <= 0:
|
||||
break
|
||||
|
||||
# Sample from remaining middle lines
|
||||
if remaining_slots > 0:
|
||||
middle_start = self.config.keep_first_lines
|
||||
middle_end = total - self.config.keep_last_lines
|
||||
|
||||
for i in range(middle_start, middle_end, self.config.sample_every_n_lines):
|
||||
if i not in selected_indices:
|
||||
selected_indices.add(i)
|
||||
remaining_slots -= 1
|
||||
if remaining_slots <= 0:
|
||||
break
|
||||
|
||||
# Sort by line number and return
|
||||
selected = sorted(selected_indices)
|
||||
return [(i, original_lines[i]) for i in selected]
|
||||
|
||||
def _format_output(self, selected: list[tuple[int, str]], total_lines: int) -> str:
|
||||
"""Format selected lines with ellipsis markers."""
|
||||
if not selected:
|
||||
return f"[{total_lines} lines omitted]"
|
||||
|
||||
output_lines: list[str] = []
|
||||
prev_idx = -1
|
||||
|
||||
for idx, line in selected:
|
||||
# Add ellipsis if there's a gap
|
||||
if prev_idx >= 0 and idx - prev_idx > 1:
|
||||
gap = idx - prev_idx - 1
|
||||
output_lines.append(f"[... {gap} lines omitted ...]")
|
||||
|
||||
output_lines.append(line)
|
||||
prev_idx = idx
|
||||
|
||||
# Add trailing ellipsis if needed
|
||||
if selected and selected[-1][0] < total_lines - 1:
|
||||
gap = total_lines - selected[-1][0] - 1
|
||||
output_lines.append(f"[... {gap} lines omitted ...]")
|
||||
|
||||
return "\n".join(output_lines)
|
||||
|
||||
def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
|
||||
"""Store original in CCR for later retrieval."""
|
||||
try:
|
||||
from ..cache.compression_store import get_compression_store
|
||||
|
||||
store = get_compression_store()
|
||||
return store.store(
|
||||
original,
|
||||
compressed,
|
||||
original_item_count=original_count,
|
||||
)
|
||||
except ImportError:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextCompressionResult:
|
||||
"""Result of text compression."""
|
||||
|
||||
compressed: str
|
||||
original: str
|
||||
original_line_count: int
|
||||
compressed_line_count: int
|
||||
compression_ratio: float
|
||||
cache_key: str | None = None
|
||||
|
||||
@property
|
||||
def tokens_saved_estimate(self) -> int:
|
||||
"""Estimate tokens saved."""
|
||||
chars_saved = len(self.original) - len(self.compressed)
|
||||
return max(0, chars_saved // 4)
|
||||
|
||||
@property
|
||||
def lines_omitted(self) -> int:
|
||||
"""Number of lines omitted."""
|
||||
return self.original_line_count - self.compressed_line_count
|
||||
|
|
@ -5,7 +5,7 @@ the keyword tables exposed by the Rust `headroom._core.signals` module.
|
|||
These tests pin:
|
||||
|
||||
* The shim re-exports the legacy frozenset and Pattern names callers
|
||||
rely on (text_compressor, search_compressor, intelligent_context).
|
||||
rely on (search_compressor, intelligent_context).
|
||||
* Two bug fixes from the Rust port land in the Python regex too:
|
||||
1. ERROR_PATTERN now matches abort/timeout/denied/rejected (was a
|
||||
drift between ERROR_KEYWORDS and the compiled regex).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Tests for text-based compressors (coding task support).
|
||||
|
||||
Tests content detection, search compressor, log compressor, and text compressor.
|
||||
Tests content detection, search compressor, and log compressor.
|
||||
"""
|
||||
|
||||
from headroom.transforms import (
|
||||
|
|
@ -11,7 +11,6 @@ from headroom.transforms import (
|
|||
SearchCompressorConfig,
|
||||
detect_content_type,
|
||||
)
|
||||
from headroom.transforms.text_compressor import TextCompressor, TextCompressorConfig
|
||||
|
||||
|
||||
class TestContentDetector:
|
||||
|
|
@ -241,55 +240,14 @@ INFO: Done
|
|||
assert result.compression_ratio == 1.0
|
||||
|
||||
|
||||
class TestTextCompressor:
|
||||
"""Tests for generic text compression."""
|
||||
|
||||
def test_compress_large_text(self):
|
||||
"""Large text is compressed."""
|
||||
lines = [f"Line {i}: This is some content that repeats" for i in range(200)]
|
||||
content = "\n".join(lines)
|
||||
|
||||
compressor = TextCompressor()
|
||||
result = compressor.compress(content)
|
||||
|
||||
assert result.compression_ratio < 0.5
|
||||
assert "Line 0:" in result.compressed # First line kept
|
||||
assert "lines omitted" in result.compressed
|
||||
|
||||
def test_preserves_context_matches(self):
|
||||
"""Lines matching context are preserved."""
|
||||
lines = [f"Line {i}: normal content" for i in range(100)]
|
||||
lines[50] = "Line 50: IMPORTANT error message"
|
||||
content = "\n".join(lines)
|
||||
|
||||
compressor = TextCompressor(
|
||||
config=TextCompressorConfig(
|
||||
anchor_keywords=["error", "important"],
|
||||
)
|
||||
)
|
||||
result = compressor.compress(content, context="find errors")
|
||||
|
||||
assert "IMPORTANT error message" in result.compressed
|
||||
|
||||
def test_small_text_unchanged(self):
|
||||
"""Small text passes through unchanged."""
|
||||
content = "Short text\nthat fits easily"
|
||||
|
||||
compressor = TextCompressor()
|
||||
result = compressor.compress(content)
|
||||
|
||||
assert result.compression_ratio == 1.0
|
||||
assert result.compressed == content
|
||||
|
||||
|
||||
class TestSmartCrusherTextIntegration:
|
||||
"""Tests for SmartCrusher behavior with different content types.
|
||||
|
||||
NOTE: SmartCrusher is designed for JSON compression only.
|
||||
Plain text content (search results, logs, etc.) passes through UNCHANGED.
|
||||
|
||||
Text compression utilities (SearchCompressor, LogCompressor, TextCompressor)
|
||||
are available as standalone tools for applications to use explicitly.
|
||||
Text compression utilities (SearchCompressor, LogCompressor) are
|
||||
available as standalone tools for applications to use explicitly.
|
||||
This is intentional - text compression is opt-in, not automatic.
|
||||
"""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue