diff --git a/headroom/memory/adapters/__init__.py b/headroom/memory/adapters/__init__.py index 160823f33..d0dc11bb8 100644 --- a/headroom/memory/adapters/__init__.py +++ b/headroom/memory/adapters/__init__.py @@ -21,21 +21,31 @@ from headroom.memory.adapters.graph import InMemoryGraphStore from headroom.memory.adapters.sqlite import SQLiteMemoryStore # Check for optional dependencies availability -try: - from headroom.memory.adapters.hnsw import HNSW_AVAILABLE -except ImportError: - HNSW_AVAILABLE = False +# Note: We don't import from hnsw.py here because hnswlib may crash with +# "Illegal instruction" on CPUs without required instructions (e.g., AVX). +# Instead, we check lazily when HNSWVectorIndex is actually used. +# HNSW_AVAILABLE is handled through __getattr__ to ensure lazy checking. # Lazy imports for optional adapters +_HNSW_AVAILABLE: bool | None = None # Internal cache for HNSW_AVAILABLE _HNSWVectorIndex = None _LocalEmbedder = None _OpenAIEmbedder = None _OllamaEmbedder = None -def __getattr__(name: str) -> type: +def __getattr__(name: str) -> type | bool: """Lazy import for optional adapters.""" global _HNSWVectorIndex, _LocalEmbedder, _OpenAIEmbedder, _OllamaEmbedder + global _HNSW_AVAILABLE + + if name == "HNSW_AVAILABLE": + # Lazily check hnswlib availability + if _HNSW_AVAILABLE is None: + from headroom.memory.adapters.hnsw import _check_hnswlib_available + + _HNSW_AVAILABLE = _check_hnswlib_available() + return _HNSW_AVAILABLE if name == "HNSWVectorIndex": if _HNSWVectorIndex is None: diff --git a/headroom/memory/adapters/hnsw.py b/headroom/memory/adapters/hnsw.py index 22bc4f4a2..33f95eeb4 100644 --- a/headroom/memory/adapters/hnsw.py +++ b/headroom/memory/adapters/hnsw.py @@ -22,18 +22,45 @@ from typing import TYPE_CHECKING, Any import numpy as np -# hnswlib is optional - may not compile on all platforms -try: - import hnswlib - - HNSW_AVAILABLE = True -except ImportError: - hnswlib = None # type: ignore[assignment] - HNSW_AVAILABLE = False - from ..models import Memory, ScopeLevel from ..ports import VectorFilter, VectorSearchResult +# hnswlib is optional - may not compile on all platforms +# NOTE: We don't import hnswlib at module level because it can crash with SIGILL +# (Illegal Instruction) on CPUs without required AVX instructions. The crash +# happens at the C level before Python's try/except can catch it. +# Instead, we import lazily when HNSWVectorIndex is actually instantiated. +hnswlib: Any = None # Will be imported lazily +HNSW_AVAILABLE: bool | None = None # None = not yet checked, True/False = checked + + +def _check_hnswlib_available() -> bool: + """Check if hnswlib is available, importing it lazily. + + Returns: + True if hnswlib is available and working. + + Note: + This function caches the result in HNSW_AVAILABLE. + On CPUs without AVX support, importing hnswlib may crash + the process with SIGILL before we can catch the error. + """ + global hnswlib, HNSW_AVAILABLE + + if HNSW_AVAILABLE is not None: + return HNSW_AVAILABLE + + try: + import hnswlib as _hnswlib + + hnswlib = _hnswlib + HNSW_AVAILABLE = True + except ImportError: + HNSW_AVAILABLE = False + + return HNSW_AVAILABLE + + if TYPE_CHECKING: pass @@ -187,12 +214,13 @@ class HNSWVectorIndex: ValueError: If auto_save is True but save_path is not provided. ImportError: If hnswlib is not installed. """ - if not HNSW_AVAILABLE: + if not _check_hnswlib_available(): raise ImportError( "hnswlib is required for HNSWVectorIndex. " "Install with: pip install hnswlib\n" "Note: hnswlib requires C++ compilation and may not be " - "available on all platforms." + "available on all platforms (crashes with SIGILL on CPUs " + "without AVX support)." ) if auto_save and save_path is None: @@ -208,7 +236,8 @@ class HNSWVectorIndex: # Initialize HNSW index with cosine similarity # hnswlib uses 'cosine' space which internally normalizes vectors - self._index = hnswlib.Index(space="cosine", dim=dimension) + # Note: hnswlib is guaranteed non-None here due to _check_hnswlib_available() above + self._index = hnswlib.Index(space="cosine", dim=dimension) # type: ignore[union-attr] self._index.init_index( max_elements=max_elements, ef_construction=ef_construction, @@ -729,7 +758,7 @@ class HNSWVectorIndex: self._ef_search = meta_data["ef_search"] # Create new index and load from file - self._index = hnswlib.Index(space="cosine", dim=self._dimension) + self._index = hnswlib.Index(space="cosine", dim=self._dimension) # type: ignore[union-attr] self._index.load_index( str(hnsw_path), max_elements=self._max_elements, @@ -757,7 +786,7 @@ class HNSWVectorIndex: """Clear all entries from the index.""" with self._lock: # Reinitialize the index - self._index = hnswlib.Index(space="cosine", dim=self._dimension) + self._index = hnswlib.Index(space="cosine", dim=self._dimension) # type: ignore[union-attr] self._index.init_index( max_elements=self._max_elements, ef_construction=self._ef_construction, diff --git a/headroom/transforms/__init__.py b/headroom/transforms/__init__.py index 43ffea900..f0093d5e9 100644 --- a/headroom/transforms/__init__.py +++ b/headroom/transforms/__init__.py @@ -11,6 +11,7 @@ from .anchor_selector import ( from .base import Transform from .cache_aligner import CacheAligner from .content_detector import ContentType, DetectionResult, detect_content_type +from .diff_compressor import DiffCompressionResult, DiffCompressor, DiffCompressorConfig from .intelligent_context import ContextStrategy, IntelligentContextManager from .log_compressor import LogCompressionResult, LogCompressor, LogCompressorConfig from .pipeline import TransformPipeline @@ -84,6 +85,9 @@ __all__ = [ "LogCompressor", "LogCompressorConfig", "LogCompressionResult", + "DiffCompressor", + "DiffCompressorConfig", + "DiffCompressionResult", "TextCompressor", "TextCompressorConfig", "TextCompressionResult", diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py index ee90a114e..f0136b3a8 100644 --- a/headroom/transforms/content_router.py +++ b/headroom/transforms/content_router.py @@ -461,6 +461,7 @@ class ContentRouter(Transform): self._smart_crusher: Any = None self._search_compressor: Any = None self._log_compressor: Any = None + self._diff_compressor: Any = None self._llmlingua: Any = None self._text_compressor: Any = None self._image_optimizer: Any = None @@ -772,6 +773,15 @@ class ContentRouter(Transform): result.compressed_line_count, ) + elif strategy == CompressionStrategy.DIFF: + compressor = self._get_diff_compressor() + if compressor: + result = compressor.compress(content, context=context) + compressed, compressed_tokens = ( + result.compressed, + result.compressed_line_count, + ) + elif strategy == CompressionStrategy.LLMLINGUA: compressed, compressed_tokens = self._try_llmlingua(content, context) @@ -907,6 +917,17 @@ class ContentRouter(Transform): logger.debug("LogCompressor not available") return self._log_compressor + def _get_diff_compressor(self) -> Any: + """Get DiffCompressor (lazy load).""" + if self._diff_compressor is None: + try: + from .diff_compressor import DiffCompressor + + self._diff_compressor = DiffCompressor() + except ImportError: + logger.debug("DiffCompressor not available") + return self._diff_compressor + def _get_llmlingua(self) -> Any: """Get LLMLinguaCompressor (lazy load).""" if self._llmlingua is None: diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.py new file mode 100644 index 000000000..91b6aaf49 --- /dev/null +++ b/headroom/transforms/diff_compressor.py @@ -0,0 +1,615 @@ +"""Git diff output compressor for unified diff format. + +This module compresses git diff output which can be very verbose with +many context lines. Typical compression: 3-10x. + +Supported formats: +- Unified diff format (git diff, diff -u) +- Combined diff format (merge conflicts) + +Compression Strategy: +1. Parse unified diff format into file sections and hunks +2. Always keep file headers (diff --git, ---, +++) +3. Always keep ALL actual changes (+/- lines) +4. Reduce context lines (` ` prefix) to configurable max +5. If too many hunks, keep first N and summarize rest +6. Add summary at end + +Key Patterns to Preserve: +- All additions (+) +- All deletions (-) +- Hunk headers (@@ ... @@) +- File headers (diff --git, ---, +++) +- Context around changes (limited) +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +@dataclass +class DiffHunk: + """A single hunk within a diff file.""" + + header: str # @@ -start,count +start,count @@ optional function + lines: list[str] # All lines in the hunk + additions: int = 0 + deletions: int = 0 + context_lines: int = 0 + score: float = 0.0 # Relevance score for context-aware compression + + @property + def change_count(self) -> int: + """Total number of actual changes (additions + deletions).""" + return self.additions + self.deletions + + +@dataclass +class DiffFile: + """A single file's diff.""" + + header: str # diff --git a/... b/... + old_file: str # --- a/... + new_file: str # +++ b/... + hunks: list[DiffHunk] = field(default_factory=list) + is_binary: bool = False + is_new_file: bool = False + is_deleted_file: bool = False + is_renamed: bool = False + + @property + def total_additions(self) -> int: + return sum(h.additions for h in self.hunks) + + @property + def total_deletions(self) -> int: + return sum(h.deletions for h in self.hunks) + + +@dataclass +class DiffCompressorConfig: + """Configuration for diff compression.""" + + # Context line limits + max_context_lines: int = 2 # Reduce from default 3 lines before/after changes + + # Hunk limits + max_hunks_per_file: int = 10 + + # File limits + max_files: int = 20 + + # Change preservation + always_keep_additions: bool = True # Always keep + lines + always_keep_deletions: bool = True # Always keep - lines + + # CCR integration + enable_ccr: bool = True + min_lines_for_ccr: int = 50 + + +@dataclass +class DiffCompressionResult: + """Result of diff compression.""" + + compressed: str + original_line_count: int + compressed_line_count: int + files_affected: int + additions: int + deletions: int + hunks_kept: int + hunks_removed: int + cache_key: str | None = None + + @property + def compression_ratio(self) -> float: + """Ratio of compressed to original (lower is better compression).""" + if self.original_line_count == 0: + return 1.0 + return self.compressed_line_count / self.original_line_count + + @property + def tokens_saved_estimate(self) -> int: + """Estimate tokens saved (rough: 1 token per 4 chars).""" + # Use line counts as proxy for chars + lines_saved = self.original_line_count - self.compressed_line_count + # Estimate ~40 chars per line average for diffs + chars_saved = lines_saved * 40 + return max(0, chars_saved // 4) + + +class DiffCompressor: + """Compresses git diff output. + + Example: + >>> compressor = DiffCompressor() + >>> result = compressor.compress(git_diff_output) + >>> print(result.compressed) # Reduced diff with summary + """ + + # Pattern for diff --git header + _DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$") + + # Pattern for --- a/file or --- /dev/null + _OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$") + + # Pattern for +++ b/file or +++ /dev/null + _NEW_FILE_PATTERN = re.compile(r"^\+\+\+ (b/(.+)|/dev/null)$") + + # Pattern for hunk header @@ -start,count +start,count @@ optional context + _HUNK_HEADER_PATTERN = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)$") + + # Pattern for binary file indication + _BINARY_PATTERN = re.compile(r"^Binary files .+ differ$") + + # Patterns for new/deleted file mode + _NEW_FILE_MODE_PATTERN = re.compile(r"^new file mode") + _DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode") + _RENAME_PATTERN = re.compile(r"^(rename|similarity|copy) ") + + # Priority patterns for context-aware hunk selection + _PRIORITY_PATTERNS = [ + re.compile(r"\b(error|exception|fail|bug|fix)\b", re.IGNORECASE), + re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE), + re.compile(r"\b(security|auth|password|secret|token)\b", re.IGNORECASE), + ] + + def __init__(self, config: DiffCompressorConfig | None = None): + """Initialize diff compressor. + + Args: + config: Compression configuration. + """ + self.config = config or DiffCompressorConfig() + + def compress(self, content: str, context: str = "") -> DiffCompressionResult: + """Compress diff output. + + Args: + content: Raw git diff output. + context: User query context for relevance scoring. + + Returns: + DiffCompressionResult with compressed output and metadata. + """ + lines = content.split("\n") + original_line_count = len(lines) + + if original_line_count < self.config.min_lines_for_ccr: + return DiffCompressionResult( + compressed=content, + original_line_count=original_line_count, + compressed_line_count=original_line_count, + files_affected=0, + additions=0, + deletions=0, + hunks_kept=0, + hunks_removed=0, + ) + + # Parse diff into structured format + diff_files = self._parse_diff(lines) + + if not diff_files: + return DiffCompressionResult( + compressed=content, + original_line_count=original_line_count, + compressed_line_count=original_line_count, + files_affected=0, + additions=0, + deletions=0, + hunks_kept=0, + hunks_removed=0, + ) + + # Score hunks by relevance + self._score_hunks(diff_files, context) + + # Compress each file's hunks + compressed_files, stats = self._compress_files(diff_files) + + # Format output + compressed_output = self._format_output(compressed_files, stats) + compressed_line_count = len(compressed_output.split("\n")) + + # Store in CCR if significant compression + cache_key = None + if self.config.enable_ccr and compressed_line_count < original_line_count * 0.8: + cache_key = self._store_in_ccr(content, compressed_output, original_line_count) + if cache_key: + compressed_output += f"\n[{original_line_count} lines compressed to {compressed_line_count}. Retrieve full diff: hash={cache_key}]" + + return DiffCompressionResult( + compressed=compressed_output, + original_line_count=original_line_count, + compressed_line_count=compressed_line_count, + files_affected=stats["files_affected"], + additions=stats["total_additions"], + deletions=stats["total_deletions"], + hunks_kept=stats["hunks_kept"], + hunks_removed=stats["hunks_removed"], + cache_key=cache_key, + ) + + def _parse_diff(self, lines: list[str]) -> list[DiffFile]: + """Parse diff content into structured format. + + Args: + lines: Lines of diff content. + + Returns: + List of DiffFile objects. + """ + diff_files: list[DiffFile] = [] + current_file: DiffFile | None = None + current_hunk: DiffHunk | None = None + i = 0 + + while i < len(lines): + line = lines[i] + + # Check for diff --git header (new file section) + if self._DIFF_GIT_PATTERN.match(line): + # Save previous hunk and file + if current_hunk and current_file: + current_file.hunks.append(current_hunk) + if current_file: + diff_files.append(current_file) + + current_file = DiffFile( + header=line, + old_file="", + new_file="", + ) + current_hunk = None + i += 1 + continue + + # Check for file mode indicators + if current_file: + if self._NEW_FILE_MODE_PATTERN.match(line): + current_file.is_new_file = True + elif self._DELETED_FILE_MODE_PATTERN.match(line): + current_file.is_deleted_file = True + elif self._RENAME_PATTERN.match(line): + current_file.is_renamed = True + elif self._BINARY_PATTERN.match(line): + current_file.is_binary = True + + # Check for --- a/file + if self._OLD_FILE_PATTERN.match(line): + if current_file: + current_file.old_file = line + i += 1 + continue + + # Check for +++ b/file + if self._NEW_FILE_PATTERN.match(line): + if current_file: + current_file.new_file = line + i += 1 + continue + + # Check for hunk header + if self._HUNK_HEADER_PATTERN.match(line): + # Save previous hunk + if current_hunk and current_file: + current_file.hunks.append(current_hunk) + + current_hunk = DiffHunk( + header=line, + lines=[], + ) + i += 1 + continue + + # Process hunk content lines + if current_hunk is not None: + if line.startswith("+") and not line.startswith("+++"): + current_hunk.additions += 1 + current_hunk.lines.append(line) + elif line.startswith("-") and not line.startswith("---"): + current_hunk.deletions += 1 + current_hunk.lines.append(line) + elif line.startswith(" ") or line == "": + current_hunk.context_lines += 1 + current_hunk.lines.append(line) + else: + # Other line (e.g., "\ No newline at end of file") + current_hunk.lines.append(line) + + i += 1 + + # Save final hunk and file + if current_hunk and current_file: + current_file.hunks.append(current_hunk) + if current_file: + diff_files.append(current_file) + + return diff_files + + def _score_hunks(self, diff_files: list[DiffFile], context: str) -> None: + """Score hunks by relevance to context. + + Args: + diff_files: Parsed diff files. + context: User query context. + """ + context_lower = context.lower() + context_words = set(context_lower.split()) if context else set() + + for diff_file in diff_files: + for hunk in diff_file.hunks: + score = 0.0 + + # Base score from change count (more changes = more important) + score += min(0.3, hunk.change_count * 0.03) + + hunk_content = "\n".join(hunk.lines).lower() + + # Score by context word overlap + for word in context_words: + if len(word) > 2 and word in hunk_content: + score += 0.2 + + # Boost for priority patterns + for pattern in self._PRIORITY_PATTERNS: + if pattern.search(hunk_content): + score += 0.3 + break + + hunk.score = min(1.0, score) + + def _compress_files(self, diff_files: list[DiffFile]) -> tuple[list[DiffFile], dict[str, int]]: + """Compress hunks in each file. + + Args: + diff_files: Parsed diff files. + + Returns: + Tuple of (compressed files, stats dict). + """ + stats = { + "files_affected": 0, + "total_additions": 0, + "total_deletions": 0, + "hunks_kept": 0, + "hunks_removed": 0, + } + + # Limit files if too many + if len(diff_files) > self.config.max_files: + # Sort by total changes (most changes first) + diff_files = sorted( + diff_files, + key=lambda f: f.total_additions + f.total_deletions, + reverse=True, + ) + diff_files = diff_files[: self.config.max_files] + + compressed_files: list[DiffFile] = [] + + for diff_file in diff_files: + stats["files_affected"] += 1 + stats["total_additions"] += diff_file.total_additions + stats["total_deletions"] += diff_file.total_deletions + + # Compress hunks within file + compressed_hunks = self._compress_hunks(diff_file.hunks) + + stats["hunks_kept"] += len(compressed_hunks) + stats["hunks_removed"] += len(diff_file.hunks) - len(compressed_hunks) + + # Create compressed file with reduced context in hunks + new_file = DiffFile( + header=diff_file.header, + old_file=diff_file.old_file, + new_file=diff_file.new_file, + hunks=compressed_hunks, + is_binary=diff_file.is_binary, + is_new_file=diff_file.is_new_file, + is_deleted_file=diff_file.is_deleted_file, + is_renamed=diff_file.is_renamed, + ) + compressed_files.append(new_file) + + return compressed_files, stats + + def _compress_hunks(self, hunks: list[DiffHunk]) -> list[DiffHunk]: + """Compress hunks by reducing context and limiting count. + + Args: + hunks: List of hunks to compress. + + Returns: + Compressed list of hunks. + """ + if not hunks: + return [] + + # Sort by score if we need to limit + if len(hunks) > self.config.max_hunks_per_file: + # Keep first and last hunks (often important) + first_hunk = hunks[0] + last_hunk = hunks[-1] if len(hunks) > 1 else None + + # Sort middle hunks by score + middle_hunks = sorted( + hunks[1:-1] if last_hunk else [], key=lambda h: h.score, reverse=True + ) + + # Take top scoring middle hunks + remaining_slots = ( + self.config.max_hunks_per_file - 2 + if last_hunk + else self.config.max_hunks_per_file - 1 + ) + selected_middle = middle_hunks[:remaining_slots] + + # Rebuild list in original order by re-sorting by appearance + selected = [first_hunk] + selected_middle + if last_hunk: + selected.append(last_hunk) + + # Sort back to original order (using header line numbers as proxy) + hunks = sorted(selected, key=lambda h: self._extract_line_number(h.header)) + + # Reduce context in each hunk + compressed_hunks = [] + for hunk in hunks: + compressed_hunk = self._reduce_context(hunk) + compressed_hunks.append(compressed_hunk) + + return compressed_hunks + + def _extract_line_number(self, header: str) -> int: + """Extract starting line number from hunk header for sorting.""" + match = self._HUNK_HEADER_PATTERN.match(header) + if match: + return int(match.group(1)) + return 0 + + def _reduce_context(self, hunk: DiffHunk) -> DiffHunk: + """Reduce context lines while preserving all changes. + + Args: + hunk: Hunk to reduce context in. + + Returns: + New hunk with reduced context. + """ + max_context = self.config.max_context_lines + + # Identify change positions + change_positions: list[int] = [] + for i, line in enumerate(hunk.lines): + if line.startswith("+") or line.startswith("-"): + change_positions.append(i) + + if not change_positions: + # No changes, just context - keep minimal + return DiffHunk( + header=hunk.header, + lines=hunk.lines[:max_context] if hunk.lines else [], + additions=0, + deletions=0, + context_lines=min(len(hunk.lines), max_context), + score=hunk.score, + ) + + # Determine which lines to keep + keep_indices: set[int] = set() + + for pos in change_positions: + # Always keep the change line + keep_indices.add(pos) + + # Keep context before + for i in range(max(0, pos - max_context), pos): + keep_indices.add(i) + + # Keep context after + for i in range(pos + 1, min(len(hunk.lines), pos + max_context + 1)): + keep_indices.add(i) + + # Build new lines list + new_lines: list[str] = [] + additions = 0 + deletions = 0 + context_lines = 0 + + for i in sorted(keep_indices): + line = hunk.lines[i] + new_lines.append(line) + if line.startswith("+"): + additions += 1 + elif line.startswith("-"): + deletions += 1 + else: + context_lines += 1 + + return DiffHunk( + header=hunk.header, + lines=new_lines, + additions=additions, + deletions=deletions, + context_lines=context_lines, + score=hunk.score, + ) + + def _format_output(self, diff_files: list[DiffFile], stats: dict[str, int]) -> str: + """Format compressed diff files back to unified diff format. + + Args: + diff_files: Compressed diff files. + stats: Compression statistics. + + Returns: + Formatted diff string. + """ + output_lines: list[str] = [] + + for diff_file in diff_files: + # File header + output_lines.append(diff_file.header) + + # File mode indicators if present + if diff_file.is_new_file: + output_lines.append("new file mode 100644") + elif diff_file.is_deleted_file: + output_lines.append("deleted file mode 100644") + + if diff_file.is_binary: + output_lines.append("Binary files differ") + continue + + # Old/new file markers + if diff_file.old_file: + output_lines.append(diff_file.old_file) + if diff_file.new_file: + output_lines.append(diff_file.new_file) + + # Hunks + for hunk in diff_file.hunks: + output_lines.append(hunk.header) + output_lines.extend(hunk.lines) + + # Add summary + if stats["hunks_removed"] > 0 or stats["files_affected"] > 0: + summary_parts = [ + f"{stats['files_affected']} files changed", + f"+{stats['total_additions']} -{stats['total_deletions']} lines", + ] + if stats["hunks_removed"] > 0: + summary_parts.append(f"{stats['hunks_removed']} hunks omitted") + output_lines.append(f"[{', '.join(summary_parts)}]") + + return "\n".join(output_lines) + + def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None: + """Store original in CCR for later retrieval. + + Args: + original: Original diff content. + compressed: Compressed diff content. + original_count: Original line count. + + Returns: + Cache key if stored, None otherwise. + """ + try: + from ..cache.compression_store import get_compression_store + + store = get_compression_store() + return store.store( + original, + compressed, + original_item_count=original_count, + ) + except ImportError: + return None + except Exception: + return None diff --git a/tests/test_hnsw_only.py b/tests/test_hnsw_only.py index c33d9e37b..3e2c9e179 100644 --- a/tests/test_hnsw_only.py +++ b/tests/test_hnsw_only.py @@ -13,9 +13,11 @@ import pytest from headroom.memory.models import Memory from headroom.memory.ports import VectorFilter -# Check if hnswlib is available +# Check if hnswlib is available (use lazy check to avoid SIGILL on incompatible CPUs) try: - from headroom.memory.adapters.hnsw import HNSW_AVAILABLE + from headroom.memory.adapters.hnsw import _check_hnswlib_available + + HNSW_AVAILABLE = _check_hnswlib_available() except ImportError: HNSW_AVAILABLE = False diff --git a/tests/test_memory/test_hierarchical.py b/tests/test_memory/test_hierarchical.py index 0b567f0da..afe86fbd0 100644 --- a/tests/test_memory/test_hierarchical.py +++ b/tests/test_memory/test_hierarchical.py @@ -608,9 +608,11 @@ class TestIntegration: # HNSW Vector Index Tests # ============================================================================= -# Check if hnswlib is available +# Check if hnswlib is available (use lazy check to avoid SIGILL on incompatible CPUs) try: - from headroom.memory.adapters.hnsw import HNSW_AVAILABLE + from headroom.memory.adapters.hnsw import _check_hnswlib_available + + HNSW_AVAILABLE = _check_hnswlib_available() except ImportError: HNSW_AVAILABLE = False diff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.py new file mode 100644 index 000000000..323e7ca3e --- /dev/null +++ b/tests/test_transforms/test_diff_compressor.py @@ -0,0 +1,673 @@ +"""Comprehensive tests for diff_compressor.py. + +Tests cover: +1. Parsing of unified diff format +2. Context line reduction +3. Hunk selection and limiting +4. Compression ratios +5. Edge cases +""" + +from headroom.transforms.diff_compressor import ( + DiffCompressionResult, + DiffCompressor, + DiffCompressorConfig, + DiffFile, + DiffHunk, +) + + +class TestDiffParsing: + """Tests for parsing unified diff format.""" + + def test_parse_simple_diff(self): + """Simple single-file diff is parsed correctly.""" + content = """diff --git a/src/main.py b/src/main.py +--- a/src/main.py ++++ b/src/main.py +@@ -10,6 +10,7 @@ def main(): + print("hello") ++ print("world") + return 0 +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + + assert len(diff_files) == 1 + assert diff_files[0].header == "diff --git a/src/main.py b/src/main.py" + assert diff_files[0].old_file == "--- a/src/main.py" + assert diff_files[0].new_file == "+++ b/src/main.py" + assert len(diff_files[0].hunks) == 1 + assert diff_files[0].hunks[0].additions == 1 + assert diff_files[0].hunks[0].deletions == 0 + + def test_parse_multi_file_diff(self): + """Multi-file diff is parsed into separate DiffFile objects.""" + content = """diff --git a/file1.py b/file1.py +--- a/file1.py ++++ b/file1.py +@@ -1,3 +1,4 @@ + line1 ++added line + line2 +diff --git a/file2.py b/file2.py +--- a/file2.py ++++ b/file2.py +@@ -5,4 +5,3 @@ + keep +-removed + keep2 +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + + assert len(diff_files) == 2 + assert "file1.py" in diff_files[0].header + assert "file2.py" in diff_files[1].header + assert diff_files[0].hunks[0].additions == 1 + assert diff_files[0].hunks[0].deletions == 0 + assert diff_files[1].hunks[0].additions == 0 + assert diff_files[1].hunks[0].deletions == 1 + + def test_parse_multi_hunk_file(self): + """File with multiple hunks is parsed correctly.""" + content = """diff --git a/src/utils.py b/src/utils.py +--- a/src/utils.py ++++ b/src/utils.py +@@ -10,4 +10,5 @@ def helper(): + pass ++ # added comment + return True +@@ -50,3 +51,4 @@ def other(): + x = 1 ++ y = 2 + return x +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + + assert len(diff_files) == 1 + assert len(diff_files[0].hunks) == 2 + assert diff_files[0].total_additions == 2 + + def test_parse_new_file(self): + """New file diff is detected.""" + content = """diff --git a/newfile.py b/newfile.py +new file mode 100644 +--- /dev/null ++++ b/newfile.py +@@ -0,0 +1,3 @@ ++def new_func(): ++ pass ++ return None +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + + assert len(diff_files) == 1 + assert diff_files[0].is_new_file is True + assert diff_files[0].hunks[0].additions == 3 + + def test_parse_deleted_file(self): + """Deleted file diff is detected.""" + content = """diff --git a/oldfile.py b/oldfile.py +deleted file mode 100644 +--- a/oldfile.py ++++ /dev/null +@@ -1,2 +0,0 @@ +-def old_func(): +- pass +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + + assert len(diff_files) == 1 + assert diff_files[0].is_deleted_file is True + assert diff_files[0].hunks[0].deletions == 2 + + def test_parse_binary_file(self): + """Binary file diff is detected.""" + content = """diff --git a/image.png b/image.png +Binary files a/image.png and b/image.png differ +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + + assert len(diff_files) == 1 + assert diff_files[0].is_binary is True + + +class TestContextReduction: + """Tests for context line reduction.""" + + def test_reduce_context_lines(self): + """Context lines are reduced to configured maximum.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,10 +1,11 @@ + context1 + context2 + context3 + context4 ++added + context5 + context6 + context7 + context8 +""" + # Default max_context_lines is 2 + compressor = DiffCompressor( + config=DiffCompressorConfig( + max_context_lines=2, + min_lines_for_ccr=5, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # Should keep 2 context before and 2 after the +added line + # Plus the added line itself + lines = result.compressed.split("\n") + context_count = sum(1 for line in lines if line.startswith(" ")) + + # At most 4 context lines (2 before + 2 after) + assert context_count <= 4 + + def test_preserve_all_changes(self): + """All addition and deletion lines are preserved.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,10 +1,10 @@ + ctx1 + ctx2 +-removed1 ++added1 + ctx3 + ctx4 +-removed2 ++added2 + ctx5 + ctx6 +""" + compressor = DiffCompressor( + config=DiffCompressorConfig( + min_lines_for_ccr=5, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + assert "-removed1" in result.compressed + assert "-removed2" in result.compressed + assert "+added1" in result.compressed + assert "+added2" in result.compressed + + +class TestHunkSelection: + """Tests for hunk selection when limiting.""" + + def test_max_hunks_per_file(self): + """Hunks are limited to max_hunks_per_file.""" + # Create a diff with many hunks + hunks = [] + for i in range(20): + hunks.append(f"""@@ -{i * 10},3 +{i * 10},4 @@ + context ++added_{i} + more +""") + + content = f"""diff --git a/bigfile.py b/bigfile.py +--- a/bigfile.py ++++ b/bigfile.py +{"".join(hunks)}""" + + compressor = DiffCompressor( + config=DiffCompressorConfig( + max_hunks_per_file=5, + min_lines_for_ccr=10, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # Should have at most 5 hunks + hunk_count = result.compressed.count("@@") + # Each hunk has one @@ header (we count full hunk headers) + assert hunk_count <= 10 # Each hunk header appears twice @@...@@ + + def test_keeps_first_and_last_hunk(self): + """First and last hunks are preserved when limiting.""" + hunks = [] + for i in range(10): + hunks.append(f"""@@ -{i * 10},3 +{i * 10},4 @@ + context ++added_{i} + more +""") + + content = f"""diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +{"".join(hunks)}""" + + compressor = DiffCompressor( + config=DiffCompressorConfig( + max_hunks_per_file=3, + min_lines_for_ccr=10, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # First hunk (added_0) should be present + assert "+added_0" in result.compressed + # Last hunk (added_9) should be present + assert "+added_9" in result.compressed + + +class TestFileSelection: + """Tests for file selection when limiting.""" + + def test_max_files(self): + """Files are limited to max_files.""" + # Create diff with many files + files = [] + for i in range(30): + files.append(f"""diff --git a/file{i}.py b/file{i}.py +--- a/file{i}.py ++++ b/file{i}.py +@@ -1,2 +1,3 @@ + ctx ++added + ctx2 +""") + + content = "\n".join(files) + + compressor = DiffCompressor( + config=DiffCompressorConfig( + max_files=10, + min_lines_for_ccr=20, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # Count diff --git headers + file_count = result.compressed.count("diff --git") + assert file_count <= 10 + + +class TestCompressionResult: + """Tests for DiffCompressionResult properties.""" + + def test_compression_ratio_calculation(self): + """Compression ratio is calculated correctly.""" + result = DiffCompressionResult( + compressed="a\nb\nc", + original_line_count=100, + compressed_line_count=10, + files_affected=2, + additions=5, + deletions=3, + hunks_kept=2, + hunks_removed=5, + ) + + assert result.compression_ratio == 0.1 + + def test_tokens_saved_estimate(self): + """Token savings estimation works correctly.""" + result = DiffCompressionResult( + compressed="short", + original_line_count=100, + compressed_line_count=10, + files_affected=1, + additions=10, + deletions=5, + hunks_kept=1, + hunks_removed=0, + ) + + # 90 lines saved * 40 chars/line / 4 chars/token = 900 tokens + assert result.tokens_saved_estimate == 900 + + +class TestHunkScoring: + """Tests for context-aware hunk scoring.""" + + def test_score_by_context_keywords(self): + """Hunks containing context keywords get higher scores.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,3 +1,4 @@ + normal context ++normal change + more context +@@ -10,3 +11,4 @@ + error handling ++fix the bug here + return result +""" + compressor = DiffCompressor() + diff_files = compressor._parse_diff(content.split("\n")) + compressor._score_hunks(diff_files, "fix error bug") + + # Second hunk should have higher score (contains "fix" and "bug") + assert len(diff_files[0].hunks) == 2 + assert diff_files[0].hunks[1].score > diff_files[0].hunks[0].score + + def test_score_priority_patterns(self): + """Hunks with priority patterns (error, security) score higher.""" + compressor = DiffCompressor() + + hunk_normal = DiffHunk( + header="@@ -1,1 +1,2 @@", + lines=["+normal change"], + additions=1, + ) + hunk_error = DiffHunk( + header="@@ -10,1 +10,2 @@", + lines=["+fix critical error"], + additions=1, + ) + + diff_file = DiffFile( + header="diff --git a/f.py b/f.py", + old_file="--- a/f.py", + new_file="+++ b/f.py", + hunks=[hunk_normal, hunk_error], + ) + + compressor._score_hunks([diff_file], "") + + assert hunk_error.score > hunk_normal.score + + +class TestSmallDiffPassthrough: + """Tests for small diff passthrough behavior.""" + + def test_small_diff_unchanged(self): + """Diffs smaller than threshold pass through unchanged.""" + content = """diff --git a/small.py b/small.py +--- a/small.py ++++ b/small.py +@@ -1,2 +1,3 @@ + line1 ++added + line2 +""" + compressor = DiffCompressor( + config=DiffCompressorConfig( + min_lines_for_ccr=100, # High threshold + ) + ) + result = compressor.compress(content) + + # Should be unchanged + assert result.compressed == content + assert result.compression_ratio == 1.0 + + +class TestOutputFormatting: + """Tests for output formatting.""" + + def test_summary_line_added(self): + """Summary line is added at end of compressed diff.""" + # Large diff that will be compressed + hunks = [] + for i in range(15): + hunks.append(f"""@@ -{i * 10},5 +{i * 10},6 @@ + ctx1 + ctx2 ++added_{i} + ctx3 + ctx4 +""") + + content = f"""diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +{"".join(hunks)}""" + + compressor = DiffCompressor( + config=DiffCompressorConfig( + max_hunks_per_file=5, + min_lines_for_ccr=10, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # Should have summary at end + assert "files changed" in result.compressed + assert "hunks omitted" in result.compressed + + def test_preserves_diff_format(self): + """Output preserves valid unified diff format.""" + content = """diff --git a/test.py b/test.py +--- a/test.py ++++ b/test.py +@@ -1,3 +1,4 @@ + def test(): ++ # new comment + pass + return True +""" + compressor = DiffCompressor( + config=DiffCompressorConfig( + min_lines_for_ccr=5, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # Should have all standard diff markers + assert "diff --git" in result.compressed + assert "---" in result.compressed + assert "+++" in result.compressed + assert "@@" in result.compressed + + +class TestEdgeCases: + """Tests for edge cases and boundary conditions.""" + + def test_empty_input(self): + """Empty input is handled gracefully.""" + compressor = DiffCompressor() + result = compressor.compress("") + + assert result.compressed == "" + assert result.compression_ratio == 1.0 + + def test_non_diff_input(self): + """Non-diff input passes through unchanged.""" + content = "This is not a diff\nJust regular text" + compressor = DiffCompressor() + result = compressor.compress(content) + + # Should pass through (no diff --git found) + assert result.compressed == content + + def test_unicode_content(self): + """Unicode characters in diff are handled.""" + content = """diff --git a/i18n.py b/i18n.py +--- a/i18n.py ++++ b/i18n.py +@@ -1,2 +1,3 @@ + msg = "hello" ++msg_ja = "こんにちは" + return msg +""" + compressor = DiffCompressor() + result = compressor.compress(content) + + assert "こんにちは" in result.compressed + + def test_no_newline_at_eof(self): + """Handles 'No newline at end of file' indicator.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,2 +1,2 @@ + line1 +-line2 +\\ No newline at end of file ++line2_modified +\\ No newline at end of file +""" + compressor = DiffCompressor() + result = compressor.compress(content) + + # Should not crash and preserve the indicator + assert "No newline" in result.compressed or "-line2" in result.compressed + + def test_empty_hunks(self): + """Files with no actual hunks are handled.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +""" + compressor = DiffCompressor() + result = compressor.compress(content) + + # Should not crash + assert result.compressed is not None + + +class TestDiffHunkDataclass: + """Tests for DiffHunk dataclass.""" + + def test_change_count_property(self): + """change_count returns sum of additions and deletions.""" + hunk = DiffHunk( + header="@@ -1,5 +1,6 @@", + lines=["+a", "+b", "-c", " ctx"], + additions=2, + deletions=1, + ) + assert hunk.change_count == 3 + + def test_default_values(self): + """DiffHunk default values are correct.""" + hunk = DiffHunk(header="@@", lines=[]) + assert hunk.additions == 0 + assert hunk.deletions == 0 + assert hunk.context_lines == 0 + assert hunk.score == 0.0 + + +class TestDiffFileDataclass: + """Tests for DiffFile dataclass.""" + + def test_total_additions_property(self): + """total_additions sums across all hunks.""" + hunk1 = DiffHunk(header="@@", lines=[], additions=3) + hunk2 = DiffHunk(header="@@", lines=[], additions=5) + diff_file = DiffFile( + header="diff --git", + old_file="---", + new_file="+++", + hunks=[hunk1, hunk2], + ) + assert diff_file.total_additions == 8 + + def test_total_deletions_property(self): + """total_deletions sums across all hunks.""" + hunk1 = DiffHunk(header="@@", lines=[], deletions=2) + hunk2 = DiffHunk(header="@@", lines=[], deletions=4) + diff_file = DiffFile( + header="diff --git", + old_file="---", + new_file="+++", + hunks=[hunk1, hunk2], + ) + assert diff_file.total_deletions == 6 + + +class TestConfigOptions: + """Tests for configuration options.""" + + def test_max_context_lines_config(self): + """max_context_lines configuration controls context reduction.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,10 +1,11 @@ + c1 + c2 + c3 + c4 + c5 ++added + c6 + c7 + c8 + c9 + c10 +""" + # With max_context_lines=1 + compressor = DiffCompressor( + config=DiffCompressorConfig( + max_context_lines=1, + min_lines_for_ccr=5, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + # Count context lines (lines starting with space) + context_count = sum(1 for line in result.compressed.split("\n") if line.startswith(" ")) + + # Should have at most 2 context lines (1 before + 1 after) + assert context_count <= 2 + + def test_always_keep_additions_default(self): + """Additions are always kept by default.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,3 +1,5 @@ + ctx ++add1 ++add2 + ctx +""" + compressor = DiffCompressor( + config=DiffCompressorConfig( + always_keep_additions=True, + min_lines_for_ccr=2, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + assert "+add1" in result.compressed + assert "+add2" in result.compressed + + def test_always_keep_deletions_default(self): + """Deletions are always kept by default.""" + content = """diff --git a/file.py b/file.py +--- a/file.py ++++ b/file.py +@@ -1,5 +1,3 @@ + ctx +-del1 +-del2 + ctx +""" + compressor = DiffCompressor( + config=DiffCompressorConfig( + always_keep_deletions=True, + min_lines_for_ccr=2, + enable_ccr=False, + ) + ) + result = compressor.compress(content) + + assert "-del1" in result.compressed + assert "-del2" in result.compressed