From f5f465418bbe6b2fcffc7c28e15c5ee4dc850df1 Mon Sep 17 00:00:00 2001 From: chopratejas Date: Sun, 26 Apr 2026 08:12:44 -0700 Subject: [PATCH] feat(rust): retire python diff_compressor, ship rust-only via pyo3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The python `DiffCompressor` is replaced by a thin pyo3-backed shim that delegates to `headroom._core.DiffCompressor`. There is no python implementation and no env-var fallback — the wheel is a hard import. Why now: opt-in defaults don't drive python retirement. Byte-equal parity was already proven across 27 fixtures (stage 3a); keeping a shadow python impl behind a flag is a permanent maintenance cost with no operational benefit. Stage 3b deletes ~700 lines of python parser / scorer / formatter code; the rust crate has its own coverage. Surface preserved: - `headroom.transforms.diff_compressor.DiffCompressor` — same class name, same `__init__`, same `compress(content, context)` shape. Returns python `DiffCompressionResult` dataclasses so call sites that destructure with `asdict()` work unchanged. - `DiffCompressorConfig` and `DiffCompressionResult` dataclasses kept. - Sidecar `compress_with_stats(...)` exposes the rust-only `DiffCompressorStats` (per-file hunk drops, context lines trimmed, file_mode normalizations) for observability. Removed: - Python parser / scorer / formatter (~700 lines). - Internal `DiffHunk` / `DiffFile` parser dataclasses (rust crate has parallel coverage). - 12 tests that probed `_parse_diff` / `_score_hunks` or the deleted parser dataclasses. The 29 public-API tests in `test_diff_compressor.py` remain and now exercise the rust backend through the same import path. - `HEADROOM_DIFF_COMPRESSOR_BACKEND` env var — no longer meaningful. Build: - `scripts/build_rust_extension.sh` runs `maturin develop` and symlinks the built `.so` into `headroom/` so `import headroom._core` resolves past the in-tree package shadowing the maturin overlay. - `.gitignore` excludes the symlinks and allowlists the build script. Tests: - 544 transforms pass; 33 rust-vs-fixture parity tests guard the pyo3 bridge against regressions (`tests/test_transforms/test_diff_compressor_rust_parity.py`). - Mypy clean. --- .gitignore | 9 + crates/headroom-py/src/lib.rs | 367 +++++++- headroom/transforms/content_router.py | 11 +- headroom/transforms/diff_compressor.py | 804 ++---------------- scripts/build_rust_extension.sh | 45 + tests/test_transforms/test_diff_compressor.py | 245 +----- .../test_diff_compressor_rust_parity.py | 119 +++ 7 files changed, 637 insertions(+), 963 deletions(-) create mode 100755 scripts/build_rust_extension.sh create mode 100644 tests/test_transforms/test_diff_compressor_rust_parity.py diff --git a/.gitignore b/.gitignore index 51e79e0db..af93df6fe 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ scripts/* !scripts/fixtures/ !scripts/fixtures/*.json !scripts/record_fixtures.py +!scripts/build_rust_extension.sh # Rust / Cargo build artifacts /target/ @@ -223,3 +224,11 @@ headroom-managed/ # uv lockfile: regenerated locally; not committed uv.lock + +# Rust extension `.so` symlinks placed by `scripts/build_rust_extension.sh` +# into the `headroom/` package dir for local development. The real binary +# lives in `crates/headroom-py/python/headroom/`; this is the dev overlay +# that lets `import headroom._core` resolve when the source `headroom/` +# package shadows the maturin overlay on sys.path. +/headroom/_core.*.so +/headroom/_core.so diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs index 70a693c98..0e51e1ba9 100644 --- a/crates/headroom-py/src/lib.rs +++ b/crates/headroom-py/src/lib.rs @@ -1,15 +1,380 @@ //! PyO3 bindings for headroom-core. Exposed to Python as `headroom._core`. +//! +//! # Stage 3b — diff_compressor bridge +//! +//! The `DiffCompressor` family is exported here so the Python +//! `ContentRouter` can route to the Rust implementation in-process via +//! PyO3 instead of running the Python port. Backend selection happens in +//! `headroom.transforms._rust_diff_compressor.RustBackedDiffCompressor`, +//! which mirrors the Python `DiffCompressor` API one-for-one (so callers +//! don't notice the swap). +//! +//! Why in-process: ContentRouter compresses on the proxy's hot path. Any +//! IPC / subprocess / RPC bridge would dominate the cost we're trying to +//! save. PyO3 calls cost ~microseconds; staying in-process is ~free. +use std::collections::BTreeMap; + +use headroom_core::transforms::{ + DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats, +}; use pyo3::prelude::*; -/// Return the identity string from headroom-core. +/// Identity stub used by the Python smoke test to verify linkage. #[pyfunction] fn hello() -> &'static str { headroom_core::hello() } +// ─── DiffCompressorConfig ────────────────────────────────────────────────── + +/// Mirror of `headroom.transforms.diff_compressor.DiffCompressorConfig`. +/// Defaults match Python; constructor accepts every field as a kwarg with +/// the same name and type as the Python dataclass for drop-in +/// compatibility. +#[pyclass(name = "DiffCompressorConfig", module = "headroom._core")] +#[derive(Clone)] +struct PyDiffCompressorConfig { + inner: DiffCompressorConfig, +} + +#[pymethods] +impl PyDiffCompressorConfig { + #[new] + #[pyo3(signature = ( + max_context_lines = 2, + max_hunks_per_file = 10, + max_files = 20, + always_keep_additions = true, + always_keep_deletions = true, + enable_ccr = true, + min_lines_for_ccr = 50, + min_compression_ratio_for_ccr = 0.8, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + max_context_lines: usize, + max_hunks_per_file: usize, + max_files: usize, + always_keep_additions: bool, + always_keep_deletions: bool, + enable_ccr: bool, + min_lines_for_ccr: usize, + min_compression_ratio_for_ccr: f64, + ) -> Self { + Self { + inner: DiffCompressorConfig { + max_context_lines, + max_hunks_per_file, + max_files, + always_keep_additions, + always_keep_deletions, + enable_ccr, + min_lines_for_ccr, + min_compression_ratio_for_ccr, + }, + } + } + + // Read-only field accessors mirroring the Python dataclass surface. + #[getter] + fn max_context_lines(&self) -> usize { + self.inner.max_context_lines + } + #[getter] + fn max_hunks_per_file(&self) -> usize { + self.inner.max_hunks_per_file + } + #[getter] + fn max_files(&self) -> usize { + self.inner.max_files + } + #[getter] + fn always_keep_additions(&self) -> bool { + self.inner.always_keep_additions + } + #[getter] + fn always_keep_deletions(&self) -> bool { + self.inner.always_keep_deletions + } + #[getter] + fn enable_ccr(&self) -> bool { + self.inner.enable_ccr + } + #[getter] + fn min_lines_for_ccr(&self) -> usize { + self.inner.min_lines_for_ccr + } + #[getter] + fn min_compression_ratio_for_ccr(&self) -> f64 { + self.inner.min_compression_ratio_for_ccr + } + + fn __repr__(&self) -> String { + format!( + "DiffCompressorConfig(max_context_lines={}, max_hunks_per_file={}, max_files={}, \ + always_keep_additions={}, always_keep_deletions={}, enable_ccr={}, \ + min_lines_for_ccr={}, min_compression_ratio_for_ccr={})", + self.inner.max_context_lines, + self.inner.max_hunks_per_file, + self.inner.max_files, + self.inner.always_keep_additions, + self.inner.always_keep_deletions, + self.inner.enable_ccr, + self.inner.min_lines_for_ccr, + self.inner.min_compression_ratio_for_ccr, + ) + } +} + +// ─── DiffCompressionResult ───────────────────────────────────────────────── + +/// Mirror of `headroom.transforms.diff_compressor.DiffCompressionResult`. +/// Read-only on the Python side: ContentRouter consumes fields, doesn't +/// mutate. `compression_ratio` and `tokens_saved_estimate` are exposed as +/// methods (not `@property`) — Python callers reach them via `.method()`. +/// The Python adapter wraps and re-exposes them as properties for full +/// dataclass compatibility. +#[pyclass(name = "DiffCompressionResult", module = "headroom._core")] +struct PyDiffCompressionResult { + inner: DiffCompressionResult, +} + +#[pymethods] +impl PyDiffCompressionResult { + #[getter] + fn compressed(&self) -> &str { + &self.inner.compressed + } + #[getter] + fn original_line_count(&self) -> usize { + self.inner.original_line_count + } + #[getter] + fn compressed_line_count(&self) -> usize { + self.inner.compressed_line_count + } + #[getter] + fn files_affected(&self) -> usize { + self.inner.files_affected + } + #[getter] + fn additions(&self) -> usize { + self.inner.additions + } + #[getter] + fn deletions(&self) -> usize { + self.inner.deletions + } + #[getter] + fn hunks_kept(&self) -> usize { + self.inner.hunks_kept + } + #[getter] + fn hunks_removed(&self) -> usize { + self.inner.hunks_removed + } + #[getter] + fn cache_key(&self) -> Option { + self.inner.cache_key.clone() + } + + /// Mirror of Python `@property compression_ratio`. Returns + /// `compressed_line_count / original_line_count` (1.0 if input was + /// empty). + fn compression_ratio(&self) -> f64 { + if self.inner.original_line_count == 0 { + 1.0 + } else { + self.inner.compressed_line_count as f64 / self.inner.original_line_count as f64 + } + } + + /// Mirror of Python `@property tokens_saved_estimate`. Same `chars * + /// 40 / 4` heuristic; bytes-equivalent numeric result. + fn tokens_saved_estimate(&self) -> usize { + let saved = self + .inner + .original_line_count + .saturating_sub(self.inner.compressed_line_count); + (saved * 40) / 4 + } + + fn __repr__(&self) -> String { + format!( + "DiffCompressionResult(compressed=<{} chars>, original_line_count={}, \ + compressed_line_count={}, files_affected={}, additions={}, deletions={}, \ + hunks_kept={}, hunks_removed={}, cache_key={:?})", + self.inner.compressed.len(), + self.inner.original_line_count, + self.inner.compressed_line_count, + self.inner.files_affected, + self.inner.additions, + self.inner.deletions, + self.inner.hunks_kept, + self.inner.hunks_removed, + self.inner.cache_key, + ) + } +} + +// ─── DiffCompressorStats ─────────────────────────────────────────────────── + +/// Mirror of Rust `DiffCompressorStats` — sidecar observability not +/// present in the Python dataclass. Returned only from `compress_with_stats`, +/// which the Python adapter exposes as a method on the wrapper. `Vec`s are +/// returned as Python lists; the `BTreeMap` becomes a `dict`. +#[pyclass(name = "DiffCompressorStats", module = "headroom._core")] +struct PyDiffCompressorStats { + inner: DiffCompressorStats, +} + +#[pymethods] +impl PyDiffCompressorStats { + #[getter] + fn input_lines(&self) -> usize { + self.inner.input_lines + } + #[getter] + fn output_lines(&self) -> usize { + self.inner.output_lines + } + #[getter] + fn compression_ratio(&self) -> f64 { + self.inner.compression_ratio + } + #[getter] + fn files_total(&self) -> usize { + self.inner.files_total + } + #[getter] + fn files_kept(&self) -> usize { + self.inner.files_kept + } + #[getter] + fn files_dropped(&self) -> Vec { + self.inner.files_dropped.clone() + } + #[getter] + fn hunks_total(&self) -> usize { + self.inner.hunks_total + } + #[getter] + fn hunks_kept(&self) -> usize { + self.inner.hunks_kept + } + #[getter] + fn hunks_dropped(&self) -> usize { + self.inner.hunks_dropped + } + #[getter] + fn hunks_dropped_per_file(&self) -> BTreeMap { + self.inner.hunks_dropped_per_file.clone() + } + #[getter] + fn context_lines_input(&self) -> usize { + self.inner.context_lines_input + } + #[getter] + fn context_lines_kept(&self) -> usize { + self.inner.context_lines_kept + } + #[getter] + fn context_lines_trimmed(&self) -> usize { + self.inner.context_lines_trimmed + } + #[getter] + fn largest_hunk_kept_lines(&self) -> usize { + self.inner.largest_hunk_kept_lines + } + #[getter] + fn largest_hunk_dropped_lines(&self) -> usize { + self.inner.largest_hunk_dropped_lines + } + #[getter] + fn parse_warnings(&self) -> Vec { + self.inner.parse_warnings.clone() + } + #[getter] + fn processing_duration_us(&self) -> u64 { + self.inner.processing_duration_us + } + #[getter] + fn cache_key_emitted(&self) -> bool { + self.inner.cache_key_emitted + } + #[getter] + fn ccr_skipped_reason(&self) -> Option { + self.inner.ccr_skipped_reason.clone() + } + #[getter] + fn file_mode_normalizations(&self) -> Vec<(String, String)> { + self.inner.file_mode_normalizations.clone() + } + #[getter] + fn binary_files_simplified(&self) -> Vec { + self.inner.binary_files_simplified.clone() + } +} + +// ─── DiffCompressor ──────────────────────────────────────────────────────── + +/// Mirror of `headroom.transforms.diff_compressor.DiffCompressor`. The +/// Python adapter wraps this in `RustBackedDiffCompressor` so +/// `ContentRouter` can swap backends transparently. +#[pyclass(name = "DiffCompressor", module = "headroom._core")] +struct PyDiffCompressor { + inner: DiffCompressor, +} + +#[pymethods] +impl PyDiffCompressor { + /// `__init__(config: DiffCompressorConfig | None = None)` — matches the + /// Python constructor signature one-for-one. + #[new] + #[pyo3(signature = (config = None))] + fn new(config: Option<&PyDiffCompressorConfig>) -> Self { + let cfg = config.map(|c| c.inner.clone()).unwrap_or_default(); + Self { + inner: DiffCompressor::new(cfg), + } + } + + /// `compress(content: str, context: str = "") -> DiffCompressionResult`. + /// Argument order and keyword names match the Python implementation. + #[pyo3(signature = (content, context = ""))] + fn compress(&self, content: &str, context: &str) -> PyDiffCompressionResult { + PyDiffCompressionResult { + inner: self.inner.compress(content, context), + } + } + + /// `compress_with_stats(content, context="") -> (result, stats)`. + /// Sidecar API not present in Python — exposes the Rust observability + /// struct alongside the parity-equal result. Returned as a 2-tuple to + /// keep the call site Pythonic. + #[pyo3(signature = (content, context = ""))] + fn compress_with_stats( + &self, + content: &str, + context: &str, + ) -> (PyDiffCompressionResult, PyDiffCompressorStats) { + let (result, stats) = self.inner.compress_with_stats(content, context); + ( + PyDiffCompressionResult { inner: result }, + PyDiffCompressorStats { inner: stats }, + ) + } +} + +// ─── Module init ─────────────────────────────────────────────────────────── + #[pymodule] fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(hello, m)?)?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py index 8c264fc7c..ff7278305 100644 --- a/headroom/transforms/content_router.py +++ b/headroom/transforms/content_router.py @@ -1197,14 +1197,13 @@ class ContentRouter(Transform): return self._log_compressor def _get_diff_compressor(self) -> Any: - """Get DiffCompressor (lazy load).""" + """Get DiffCompressor (lazy load). Rust-only — Python implementation + retired in Stage 3b. The wheel (`headroom._core`) is a hard import. + """ if self._diff_compressor is None: - try: - from .diff_compressor import DiffCompressor + from .diff_compressor import DiffCompressor - self._diff_compressor = DiffCompressor() - except ImportError: - logger.debug("DiffCompressor not available") + self._diff_compressor = DiffCompressor() return self._diff_compressor def _get_html_extractor(self) -> Any: diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.py index ba81902e0..7b0418f74 100644 --- a/headroom/transforms/diff_compressor.py +++ b/headroom/transforms/diff_compressor.py @@ -1,108 +1,41 @@ -"""Git diff output compressor for unified diff format. +"""Git diff output compressor — Rust-backed via PyO3. -This module compresses git diff output which can be very verbose with -many context lines. Typical compression: 3-10x. +The Python implementation has been retired (Stage 3b, 2026-04-25). All +diff compression now goes through `headroom._core.DiffCompressor` (built +from `crates/headroom-py`). The byte-equality of the two implementations +was verified against 27 recorded fixtures before the Python source was +removed; the Rust crate has its own test coverage in `crates/headroom-core/`. -Supported formats: -- Unified diff format (git diff, diff -u) -- Combined diff format (merge conflicts) +This module retains the public surface — `DiffCompressorConfig`, +`DiffCompressionResult`, `DiffCompressor` — so existing call sites +(ContentRouter, parity recorder, integrations, downstream users) keep +working unchanged. The dataclasses are still pure-Python because they +appear in dataclass-aware code paths (`asdict()`, `__dict__`, dataclass +matching). Only the `DiffCompressor` class delegates to Rust. -Compression Strategy: -1. Parse unified diff format into file sections and hunks -2. Always keep file headers (diff --git, ---, +++) -3. Always keep ALL actual changes (+/- lines) -4. Reduce context lines (` ` prefix) to configurable max -5. If too many hunks, keep first N and summarize rest -6. Add summary at end - -Key Patterns to Preserve: -- All additions (+) -- All deletions (-) -- Hunk headers (@@ ... @@) -- File headers (diff --git, ---, +++) -- Context around changes (limited) +The `headroom._core` extension is a hard import: there is no Python +fallback. Build it locally with `scripts/build_rust_extension.sh` +(wraps `maturin develop`) or install a prebuilt wheel. """ from __future__ import annotations import logging -import re -from dataclasses import dataclass, field +from dataclasses import dataclass +from typing import Any logger = logging.getLogger(__name__) -@dataclass -class DiffHunk: - """A single hunk within a diff file.""" - - header: str # @@ -start,count +start,count @@ optional function - lines: list[str] # All lines in the hunk - additions: int = 0 - deletions: int = 0 - context_lines: int = 0 - score: float = 0.0 # Relevance score for context-aware compression - - @property - def change_count(self) -> int: - """Total number of actual changes (additions + deletions).""" - return self.additions + self.deletions - - -@dataclass -class DiffFile: - """A single file's diff.""" - - header: str # diff --git a/... b/... - old_file: str # --- a/... - new_file: str # +++ b/... - hunks: list[DiffHunk] = field(default_factory=list) - is_binary: bool = False - is_new_file: bool = False - is_deleted_file: bool = False - is_renamed: bool = False - # Bug-fix: rename / copy / similarity / dissimilarity marker lines that - # were emitted by git BETWEEN `diff --git` and `--- a/` (e.g. `rename - # from old.py`, `rename to new.py`, `similarity index 95%`). Previously - # we set is_renamed=True and discarded the lines, so the output looked - # like a plain modification. Now captured verbatim and re-emitted in - # `_format_output`. - rename_lines: list[str] = field(default_factory=list) - # Bug-fix: original `new file mode ` / `deleted file mode - # ` / `Binary files X and Y differ` lines. Emit normalizes to - # `100644` / bare `Binary files differ`; capturing the originals lets - # the caller observe the loss via logging. - original_new_file_mode_line: str | None = None - original_deleted_file_mode_line: str | None = None - original_binary_line: str | None = None - - @property - def total_additions(self) -> int: - return sum(h.additions for h in self.hunks) - - @property - def total_deletions(self) -> int: - return sum(h.deletions for h in self.hunks) - - @dataclass class DiffCompressorConfig: """Configuration for diff compression.""" - # Context line limits - max_context_lines: int = 2 # Reduce from default 3 lines before/after changes - - # Hunk limits + max_context_lines: int = 2 max_hunks_per_file: int = 10 - - # File limits max_files: int = 20 - - # Change preservation - always_keep_additions: bool = True # Always keep + lines - always_keep_deletions: bool = True # Always keep - lines - - # CCR integration + always_keep_additions: bool = True + always_keep_deletions: bool = True enable_ccr: bool = True min_lines_for_ccr: int = 50 @@ -123,666 +56,85 @@ class DiffCompressionResult: @property def compression_ratio(self) -> float: - """Ratio of compressed to original (lower is better compression).""" if self.original_line_count == 0: return 1.0 return self.compressed_line_count / self.original_line_count @property def tokens_saved_estimate(self) -> int: - """Estimate tokens saved (rough: 1 token per 4 chars).""" - # Use line counts as proxy for chars lines_saved = self.original_line_count - self.compressed_line_count - # Estimate ~40 chars per line average for diffs chars_saved = lines_saved * 40 return max(0, chars_saved // 4) class DiffCompressor: - """Compresses git diff output. + """Rust-backed `DiffCompressor` (via PyO3 / `headroom._core`). - Example: - >>> compressor = DiffCompressor() - >>> result = compressor.compress(git_diff_output) - >>> print(result.compressed) # Reduced diff with summary + Same `__init__` and `compress` shape as the retired Python class — + drop-in replacement. Returns Python `DiffCompressionResult` dataclass + instances so call sites that destructure with `asdict()` or read the + `@property` fields work unchanged. """ - # Pattern for `diff --git a/X b/Y` header (regular diff). - _DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$") - - # Bug-fix (2026-04-25): merge-commit headers — `diff --combined ` - # and `diff --cc `. Both produce a single-path file diff with - # combined-diff hunk syntax (`@@@`+). Previously the parser only - # recognized `diff --git`, so merge-commit diffs from `git log -p` - # of merges fell through to "no files parsed" and were passed - # through unchanged — even though ContentRouter had routed them here - # because `--- a/` triggered the detector. - _DIFF_COMBINED_PATTERN = re.compile(r"^diff --combined (.+)$") - _DIFF_CC_PATTERN = re.compile(r"^diff --cc (.+)$") - - # Pattern for --- a/file or --- /dev/null - _OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$") - - # Pattern for +++ b/file or +++ /dev/null - _NEW_FILE_PATTERN = re.compile(r"^\+\+\+ (b/(.+)|/dev/null)$") - - # Pattern for ANY hunk header — matches both regular `@@ -A,B +C,D @@` - # and combined-diff `@@@ -A,B -C,D +E,F @@@` (and 4-way `@@@@ ... @@@@`). - # Group 1 is the @-prefix (so closing @@ can backreference). Bug-fix: - # previously hardcoded to `@@`, which silently dropped all content from - # combined-diff hunks (merge commits) — `current_hunk` was never set so - # subsequent +/- lines fell through to the no-op branch. - _HUNK_HEADER_PATTERN = re.compile(r"^(@@+) (?:-\d+(?:,\d+)? )+\+\d+(?:,\d+)? \1(.*)$") - - # Used to extract the new-file starting line number for in-order resort - # after middle-hunk selection. Works for both regular and combined diffs. - _HUNK_NEW_RANGE_PATTERN = re.compile(r"\+(\d+)") - - # Pattern for binary file indication - _BINARY_PATTERN = re.compile(r"^Binary files .+ differ$") - - # Patterns for new/deleted file mode - _NEW_FILE_MODE_PATTERN = re.compile(r"^new file mode") - _DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode") - # Bug-fix: extended to include `dissimilarity` (low-similarity rename - # marker, real git output). Previously dropped silently. - _RENAME_PATTERN = re.compile(r"^(rename|similarity|copy|dissimilarity) ") - - # Priority patterns for context-aware hunk selection (centralized) - from headroom.transforms.error_detection import PRIORITY_PATTERNS_DIFF - - _PRIORITY_PATTERNS = PRIORITY_PATTERNS_DIFF - def __init__(self, config: DiffCompressorConfig | None = None): - """Initialize diff compressor. + # Hard import — no fallback. If the wheel is missing, the user + # must build it (scripts/build_rust_extension.sh) or install a + # prebuilt one. Failing loudly here is better than silently + # degrading; see feedback memory `feedback_no_silent_fallbacks.md`. + from headroom._core import ( + DiffCompressor as _RustDiffCompressor, + ) + from headroom._core import ( + DiffCompressorConfig as _RustDiffCompressorConfig, + ) - Args: - config: Compression configuration. - """ - self.config = config or DiffCompressorConfig() + cfg = config or DiffCompressorConfig() + self.config = cfg + self._rust = _RustDiffCompressor( + _RustDiffCompressorConfig( + max_context_lines=cfg.max_context_lines, + max_hunks_per_file=cfg.max_hunks_per_file, + max_files=cfg.max_files, + always_keep_additions=cfg.always_keep_additions, + always_keep_deletions=cfg.always_keep_deletions, + enable_ccr=cfg.enable_ccr, + min_lines_for_ccr=cfg.min_lines_for_ccr, + ) + ) def compress(self, content: str, context: str = "") -> DiffCompressionResult: - """Compress diff output. - - Args: - content: Raw git diff output. - context: User query context for relevance scoring. - - Returns: - DiffCompressionResult with compressed output and metadata. - """ - lines = content.split("\n") - original_line_count = len(lines) - - if original_line_count < self.config.min_lines_for_ccr: - return DiffCompressionResult( - compressed=content, - original_line_count=original_line_count, - compressed_line_count=original_line_count, - files_affected=0, - additions=0, - deletions=0, - hunks_kept=0, - hunks_removed=0, - ) - - # Parse diff into structured format. Returns pre-diff content - # (commit headers, email headers from `git format-patch`, etc.) - # alongside the parsed files. Bug-fix: previously this content was - # silently dropped; now it's preserved verbatim before the - # compressed file sections in the output. - pre_diff_lines, diff_files = self._parse_diff(lines) - - if not diff_files: - return DiffCompressionResult( - compressed=content, - original_line_count=original_line_count, - compressed_line_count=original_line_count, - files_affected=0, - additions=0, - deletions=0, - hunks_kept=0, - hunks_removed=0, - ) - - # Score hunks by relevance - self._score_hunks(diff_files, context) - - # Compress each file's hunks - compressed_files, stats = self._compress_files(diff_files) - - # Format output, prepending pre-diff content if any. - compressed_output = self._format_output(compressed_files, stats) - if pre_diff_lines: - compressed_output = "\n".join(pre_diff_lines) + "\n" + compressed_output - - # Observability: log lossy-emit signals so prod monitoring can - # alert on outlier diffs. Counts only — no PII / file content. - self._log_loss_signals(diff_files, stats) - - compressed_line_count = len(compressed_output.split("\n")) - - # Store in CCR if significant compression - cache_key = None - if self.config.enable_ccr and compressed_line_count < original_line_count * 0.8: - cache_key = self._store_in_ccr(content, compressed_output, original_line_count) - if cache_key: - compressed_output += f"\n[{original_line_count} lines compressed to {compressed_line_count}. Retrieve full diff: hash={cache_key}]" - + r = self._rust.compress(content, context) return DiffCompressionResult( - compressed=compressed_output, - original_line_count=original_line_count, - compressed_line_count=compressed_line_count, - files_affected=stats["files_affected"], - additions=stats["total_additions"], - deletions=stats["total_deletions"], - hunks_kept=stats["hunks_kept"], - hunks_removed=stats["hunks_removed"], - cache_key=cache_key, + compressed=r.compressed, + original_line_count=r.original_line_count, + compressed_line_count=r.compressed_line_count, + files_affected=r.files_affected, + additions=r.additions, + deletions=r.deletions, + hunks_kept=r.hunks_kept, + hunks_removed=r.hunks_removed, + cache_key=r.cache_key, ) - def _parse_diff(self, lines: list[str]) -> tuple[list[str], list[DiffFile]]: - """Parse diff content into structured format. - - Args: - lines: Lines of diff content. - - Returns: - Tuple of (pre_diff_lines, diff_files). `pre_diff_lines` are any - lines before the first ``diff --git`` (commit headers from - ``git log -p``, email headers from ``git format-patch``, etc.) — - previously dropped, now preserved verbatim and re-emitted. + def compress_with_stats( + self, content: str, context: str = "" + ) -> tuple[DiffCompressionResult, Any]: + """Sidecar API exposing the Rust-only `DiffCompressorStats` struct + (per-file hunk drops, context lines trimmed, file_mode normalizations, + etc.) alongside the result. Stats is the raw PyO3 wrapper — no + Python equivalent to mirror to. Typed as `Any` because the PyO3 + class has no Python type stub. """ - diff_files: list[DiffFile] = [] - current_file: DiffFile | None = None - current_hunk: DiffHunk | None = None - pre_diff_lines: list[str] = [] - i = 0 - - while i < len(lines): - line = lines[i] - - # Check for diff --git header (new file section). Bug-fix: - # `diff --combined ` and `diff --cc ` (merge - # commits) also start a new file section — single-path, - # combined-diff hunk syntax. Treat them the same as - # `diff --git` for sectioning purposes; the path goes in - # `header` verbatim. - if ( - self._DIFF_GIT_PATTERN.match(line) - or self._DIFF_COMBINED_PATTERN.match(line) - or self._DIFF_CC_PATTERN.match(line) - ): - # Save previous hunk and file - if current_hunk and current_file: - current_file.hunks.append(current_hunk) - if current_file: - diff_files.append(current_file) - - current_file = DiffFile( - header=line, - old_file="", - new_file="", - ) - current_hunk = None - i += 1 - continue - - # Bug-fix: any line before the first `diff --git` is pre-diff - # content (commit headers, email headers). Capture for verbatim - # re-emission rather than dropping. - if current_file is None: - pre_diff_lines.append(line) - i += 1 - continue - - # Check for file mode indicators / rename / binary markers. - # Capture the original line in addition to the boolean so the - # logger / sidecar observability can surface emit-time - # normalization losses. - if current_file: - if self._NEW_FILE_MODE_PATTERN.match(line): - current_file.is_new_file = True - current_file.original_new_file_mode_line = line - elif self._DELETED_FILE_MODE_PATTERN.match(line): - current_file.is_deleted_file = True - current_file.original_deleted_file_mode_line = line - elif self._RENAME_PATTERN.match(line): - current_file.is_renamed = True - # Bug-fix: capture rename marker lines so they get - # re-emitted. Previously the boolean was set but the - # actual `rename from` / `rename to` / `similarity index` - # lines were discarded — output looked like a plain - # modification and the LLM had no way to know a file - # was renamed. - current_file.rename_lines.append(line) - elif self._BINARY_PATTERN.match(line): - current_file.is_binary = True - current_file.original_binary_line = line - - # Check for --- a/file - if self._OLD_FILE_PATTERN.match(line): - if current_file: - current_file.old_file = line - i += 1 - continue - - # Check for +++ b/file - if self._NEW_FILE_PATTERN.match(line): - if current_file: - current_file.new_file = line - i += 1 - continue - - # Check for hunk header (regular `@@` or combined-diff `@@@`+). - # Bug-fix: previously `@@@` headers didn't match, so combined - # diffs (merge commits) had ALL their content silently dropped. - if self._HUNK_HEADER_PATTERN.match(line): - # Save previous hunk - if current_hunk and current_file: - current_file.hunks.append(current_hunk) - - current_hunk = DiffHunk( - header=line, - lines=[], - ) - i += 1 - continue - - # Process hunk content lines - if current_hunk is not None: - if line.startswith("+") and not line.startswith("+++"): - current_hunk.additions += 1 - current_hunk.lines.append(line) - elif line.startswith("-") and not line.startswith("---"): - current_hunk.deletions += 1 - current_hunk.lines.append(line) - elif line.startswith(" ") or line == "": - current_hunk.context_lines += 1 - current_hunk.lines.append(line) - else: - # Other line (e.g., "\ No newline at end of file" — note - # leading backslash). Preserved here; `_reduce_context` - # force-keeps `\` lines regardless of distance from - # changes, so they survive the context trim. - current_hunk.lines.append(line) - - i += 1 - - # Save final hunk and file - if current_hunk and current_file: - current_file.hunks.append(current_hunk) - if current_file: - diff_files.append(current_file) - - return pre_diff_lines, diff_files - - def _score_hunks(self, diff_files: list[DiffFile], context: str) -> None: - """Score hunks by relevance to context. - - Args: - diff_files: Parsed diff files. - context: User query context. - """ - context_lower = context.lower() - context_words = set(context_lower.split()) if context else set() - - for diff_file in diff_files: - for hunk in diff_file.hunks: - score = 0.0 - - # Base score from change count (more changes = more important) - score += min(0.3, hunk.change_count * 0.03) - - hunk_content = "\n".join(hunk.lines).lower() - - # Score by context word overlap - for word in context_words: - if len(word) > 2 and word in hunk_content: - score += 0.2 - - # Boost for priority patterns - for pattern in self._PRIORITY_PATTERNS: - if pattern.search(hunk_content): - score += 0.3 - break - - hunk.score = min(1.0, score) - - def _compress_files(self, diff_files: list[DiffFile]) -> tuple[list[DiffFile], dict[str, int]]: - """Compress hunks in each file. - - Args: - diff_files: Parsed diff files. - - Returns: - Tuple of (compressed files, stats dict). - """ - stats = { - "files_affected": 0, - "total_additions": 0, - "total_deletions": 0, - "hunks_kept": 0, - "hunks_removed": 0, - } - - # Limit files if too many - if len(diff_files) > self.config.max_files: - # Sort by total changes (most changes first) - diff_files = sorted( - diff_files, - key=lambda f: f.total_additions + f.total_deletions, - reverse=True, - ) - diff_files = diff_files[: self.config.max_files] - - compressed_files: list[DiffFile] = [] - - for diff_file in diff_files: - stats["files_affected"] += 1 - stats["total_additions"] += diff_file.total_additions - stats["total_deletions"] += diff_file.total_deletions - - # Compress hunks within file - compressed_hunks = self._compress_hunks(diff_file.hunks) - - stats["hunks_kept"] += len(compressed_hunks) - stats["hunks_removed"] += len(diff_file.hunks) - len(compressed_hunks) - - # Create compressed file with reduced context in hunks. Bug-fix: - # previously this constructor dropped `rename_lines` and the - # `original_*_line` fields by omission — they were captured in - # the parser but never propagated to `_format_output`, so the - # emit looked exactly like the buggy old behavior. Carry them - # all through. - new_file = DiffFile( - header=diff_file.header, - old_file=diff_file.old_file, - new_file=diff_file.new_file, - hunks=compressed_hunks, - is_binary=diff_file.is_binary, - is_new_file=diff_file.is_new_file, - is_deleted_file=diff_file.is_deleted_file, - is_renamed=diff_file.is_renamed, - rename_lines=diff_file.rename_lines, - original_new_file_mode_line=diff_file.original_new_file_mode_line, - original_deleted_file_mode_line=diff_file.original_deleted_file_mode_line, - original_binary_line=diff_file.original_binary_line, - ) - compressed_files.append(new_file) - - return compressed_files, stats - - def _compress_hunks(self, hunks: list[DiffHunk]) -> list[DiffHunk]: - """Compress hunks by reducing context and limiting count. - - Args: - hunks: List of hunks to compress. - - Returns: - Compressed list of hunks. - """ - if not hunks: - return [] - - # Sort by score if we need to limit - if len(hunks) > self.config.max_hunks_per_file: - # Keep first and last hunks (often important) - first_hunk = hunks[0] - last_hunk = hunks[-1] if len(hunks) > 1 else None - - # Sort middle hunks by score - middle_hunks = sorted( - hunks[1:-1] if last_hunk else [], key=lambda h: h.score, reverse=True - ) - - # Take top scoring middle hunks - remaining_slots = ( - self.config.max_hunks_per_file - 2 - if last_hunk - else self.config.max_hunks_per_file - 1 - ) - selected_middle = middle_hunks[:remaining_slots] - - # Rebuild list in original order by re-sorting by appearance - selected = [first_hunk] + selected_middle - if last_hunk: - selected.append(last_hunk) - - # Sort back to original order (using header line numbers as proxy) - hunks = sorted(selected, key=lambda h: self._extract_line_number(h.header)) - - # Reduce context in each hunk - compressed_hunks = [] - for hunk in hunks: - compressed_hunk = self._reduce_context(hunk) - compressed_hunks.append(compressed_hunk) - - return compressed_hunks - - def _extract_line_number(self, header: str) -> int: - """Extract starting `+`-side line number from hunk header for sorting. - - Works for both regular ``@@ -A,B +C,D @@`` and combined-diff - ``@@@ -A,B -C,D +E,F @@@`` formats — finds the first ``+N`` token. - """ - match = self._HUNK_NEW_RANGE_PATTERN.search(header) - if match: - return int(match.group(1)) - return 0 - - def _reduce_context(self, hunk: DiffHunk) -> DiffHunk: - """Reduce context lines while preserving all changes. - - Args: - hunk: Hunk to reduce context in. - - Returns: - New hunk with reduced context. - """ - max_context = self.config.max_context_lines - - # Identify change positions - change_positions: list[int] = [] - for i, line in enumerate(hunk.lines): - if line.startswith("+") or line.startswith("-"): - change_positions.append(i) - - if not change_positions: - # No changes, just context - keep minimal - return DiffHunk( - header=hunk.header, - lines=hunk.lines[:max_context] if hunk.lines else [], - additions=0, - deletions=0, - context_lines=min(len(hunk.lines), max_context), - score=hunk.score, - ) - - # Determine which lines to keep - keep_indices: set[int] = set() - - for pos in change_positions: - # Always keep the change line - keep_indices.add(pos) - - # Keep context before - for i in range(max(0, pos - max_context), pos): - keep_indices.add(i) - - # Keep context after - for i in range(pos + 1, min(len(hunk.lines), pos + max_context + 1)): - keep_indices.add(i) - - # Bug-fix: ALWAYS keep `\ No newline at end of file` markers (and - # any other backslash-prefixed metadata) regardless of distance - # from a change. These are structural patch markers, not context — - # losing them breaks round-trippable patches and changes the - # semantic meaning of the trailing line in the file. - for i, line in enumerate(hunk.lines): - if line.startswith("\\"): - keep_indices.add(i) - - # Build new lines list - new_lines: list[str] = [] - additions = 0 - deletions = 0 - context_lines = 0 - - for i in sorted(keep_indices): - line = hunk.lines[i] - new_lines.append(line) - if line.startswith("+"): - additions += 1 - elif line.startswith("-"): - deletions += 1 - else: - context_lines += 1 - - return DiffHunk( - header=hunk.header, - lines=new_lines, - additions=additions, - deletions=deletions, - context_lines=context_lines, - score=hunk.score, + r, stats = self._rust.compress_with_stats(content, context) + result = DiffCompressionResult( + compressed=r.compressed, + original_line_count=r.original_line_count, + compressed_line_count=r.compressed_line_count, + files_affected=r.files_affected, + additions=r.additions, + deletions=r.deletions, + hunks_kept=r.hunks_kept, + hunks_removed=r.hunks_removed, + cache_key=r.cache_key, ) - - def _format_output(self, diff_files: list[DiffFile], stats: dict[str, int]) -> str: - """Format compressed diff files back to unified diff format. - - Args: - diff_files: Compressed diff files. - stats: Compression statistics. - - Returns: - Formatted diff string. - """ - output_lines: list[str] = [] - - for diff_file in diff_files: - # File header - output_lines.append(diff_file.header) - - # Bug-fix: emit rename / similarity / dissimilarity / copy - # marker lines immediately after `diff --git`, matching git's - # canonical output order. Previously these were captured as - # `is_renamed=True` and dropped — output looked like a plain - # modification of the old file's path. - if diff_file.rename_lines: - output_lines.extend(diff_file.rename_lines) - - # File mode indicators if present. Note: parity-bound - # normalization to `100644` (the original mode is captured in - # `original_new_file_mode_line` for observability). - if diff_file.is_new_file: - output_lines.append("new file mode 100644") - elif diff_file.is_deleted_file: - output_lines.append("deleted file mode 100644") - - if diff_file.is_binary: - output_lines.append("Binary files differ") - continue - - # Old/new file markers - if diff_file.old_file: - output_lines.append(diff_file.old_file) - if diff_file.new_file: - output_lines.append(diff_file.new_file) - - # Hunks - for hunk in diff_file.hunks: - output_lines.append(hunk.header) - output_lines.extend(hunk.lines) - - # Add summary - if stats["hunks_removed"] > 0 or stats["files_affected"] > 0: - summary_parts = [ - f"{stats['files_affected']} files changed", - f"+{stats['total_additions']} -{stats['total_deletions']} lines", - ] - if stats["hunks_removed"] > 0: - summary_parts.append(f"{stats['hunks_removed']} hunks omitted") - output_lines.append(f"[{', '.join(summary_parts)}]") - - return "\n".join(output_lines) - - def _log_loss_signals( - self, - diff_files: list[DiffFile], - stats: dict[str, int], - ) -> None: - """Surface lossy-emit signals to the logger. - - Some normalizations are parity-bound and silent in the compressed - output: file modes flatten to ``100644``, ``Binary files X and Y - differ`` simplifies to ``Binary files differ``, the ``max_files`` / - ``max_hunks_per_file`` caps drop entire files / middle hunks. Stats - about caps live in the result; emit-time normalizations live only - here. Logs let prod monitoring alert on outlier diffs without - changing the compressed bytes. - """ - mode_normalizations: list[str] = [] - binary_simplifications: list[str] = [] - for f in diff_files: - if ( - f.original_new_file_mode_line - and f.original_new_file_mode_line != "new file mode 100644" - ): - mode_normalizations.append( - f"{f.old_file} -> {f.new_file}: {f.original_new_file_mode_line}" - ) - if ( - f.original_deleted_file_mode_line - and f.original_deleted_file_mode_line != "deleted file mode 100644" - ): - mode_normalizations.append( - f"{f.old_file} -> {f.new_file}: {f.original_deleted_file_mode_line}" - ) - if f.original_binary_line and f.original_binary_line != "Binary files differ": - binary_simplifications.append(f.original_binary_line) - - if mode_normalizations: - logger.warning( - "DiffCompressor: %d file mode line(s) normalized to 100644 — " - "executable / symlink / submodule signals lost: %s", - len(mode_normalizations), - mode_normalizations, - ) - if binary_simplifications: - logger.warning( - "DiffCompressor: %d binary file detail line(s) simplified — filenames lost: %s", - len(binary_simplifications), - binary_simplifications, - ) - if stats.get("hunks_removed", 0) > 0: - logger.info( - "DiffCompressor: dropped %d hunks across %d files", - stats["hunks_removed"], - stats.get("files_affected", 0), - ) - - def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None: - """Store original in CCR for later retrieval. - - Args: - original: Original diff content. - compressed: Compressed diff content. - original_count: Original line count. - - Returns: - Cache key if stored, None otherwise. - """ - try: - from ..cache.compression_store import get_compression_store - - store = get_compression_store() - return store.store( - original, - compressed, - original_item_count=original_count, - ) - except ImportError: - return None - except Exception: - return None + return result, stats diff --git a/scripts/build_rust_extension.sh b/scripts/build_rust_extension.sh new file mode 100755 index 000000000..0e678540c --- /dev/null +++ b/scripts/build_rust_extension.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Build the Rust → Python extension (headroom._core) and link it into the +# in-tree `headroom/` package so `import headroom._core` resolves. +# +# Why a wrapper script: `maturin develop` builds the `.so` and installs it +# into the venv's site-packages, but the in-tree `headroom/` source +# directory (loaded via `pip install -e .`) shadows that on sys.path. +# Python finds `headroom/__init__.py` at the project root before reaching +# the maturin overlay, so `import headroom._core` fails. Symlinking the +# built `.so` into `headroom/` fixes the lookup with zero copies. +# +# Idempotent. Safe to run repeatedly. Requires `maturin` in PATH (i.e. +# inside the project venv). + +set -euo pipefail + +cd "$(dirname "$0")/.." + +if ! command -v maturin >/dev/null 2>&1; then + echo "error: maturin not found. Activate the venv first:" >&2 + echo " source .venv/bin/activate" >&2 + exit 1 +fi + +# Build the wheel + install into the venv site-packages. +maturin develop -m crates/headroom-py/Cargo.toml + +# Locate the built `.so`. `maturin develop` writes it under +# `crates/headroom-py/python/headroom/_core.cpython--.so`. +SO_FILE=$(find crates/headroom-py/python/headroom -maxdepth 1 \ + -name "_core.cpython-*.so" -o -name "_core.cpython-*.dylib" -o -name "_core.pyd" \ + 2>/dev/null | head -1) + +if [[ -z "$SO_FILE" ]]; then + echo "error: maturin develop succeeded but produced no _core.* binary." >&2 + exit 1 +fi + +# Symlink into the in-tree package dir. +LINK_NAME="headroom/$(basename "$SO_FILE")" +ln -sf "$(pwd)/$SO_FILE" "$LINK_NAME" +echo "linked: $LINK_NAME -> $SO_FILE" + +# Smoke-test the import to fail loudly if anything is misconfigured. +python -c "from headroom._core import DiffCompressor; print('headroom._core OK:', DiffCompressor)" diff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.py index 5be31cd14..522434767 100644 --- a/tests/test_transforms/test_diff_compressor.py +++ b/tests/test_transforms/test_diff_compressor.py @@ -1,142 +1,29 @@ -"""Comprehensive tests for diff_compressor.py. +"""Comprehensive tests for the public DiffCompressor API. Tests cover: -1. Parsing of unified diff format -2. Context line reduction -3. Hunk selection and limiting -4. Compression ratios -5. Edge cases +1. Context line reduction +2. Hunk selection and limiting +3. Compression ratios +4. Edge cases +5. Bug-fix regressions and routing-gap fixtures + +Stage 3b note (2026-04-25): the Python `DiffCompressor` implementation +was retired in favor of the Rust-backed shim (`headroom._core` via PyO3). +Tests that probed Python-only internals — `_parse_diff`, `_score_hunks`, +the `DiffHunk` / `DiffFile` parser dataclasses — were removed because +the Rust crate has its own parallel coverage in +`crates/headroom-core/tests`. Public-API tests (anything calling +`compressor.compress(...)`) are preserved unchanged: they exercise the +Rust backend through the same import path and assert the same outputs. """ from headroom.transforms.diff_compressor import ( DiffCompressionResult, DiffCompressor, DiffCompressorConfig, - DiffFile, - DiffHunk, ) -class TestDiffParsing: - """Tests for parsing unified diff format.""" - - def test_parse_simple_diff(self): - """Simple single-file diff is parsed correctly.""" - content = """diff --git a/src/main.py b/src/main.py ---- a/src/main.py -+++ b/src/main.py -@@ -10,6 +10,7 @@ def main(): - print("hello") -+ print("world") - return 0 -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - - assert len(diff_files) == 1 - assert diff_files[0].header == "diff --git a/src/main.py b/src/main.py" - assert diff_files[0].old_file == "--- a/src/main.py" - assert diff_files[0].new_file == "+++ b/src/main.py" - assert len(diff_files[0].hunks) == 1 - assert diff_files[0].hunks[0].additions == 1 - assert diff_files[0].hunks[0].deletions == 0 - - def test_parse_multi_file_diff(self): - """Multi-file diff is parsed into separate DiffFile objects.""" - content = """diff --git a/file1.py b/file1.py ---- a/file1.py -+++ b/file1.py -@@ -1,3 +1,4 @@ - line1 -+added line - line2 -diff --git a/file2.py b/file2.py ---- a/file2.py -+++ b/file2.py -@@ -5,4 +5,3 @@ - keep --removed - keep2 -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - - assert len(diff_files) == 2 - assert "file1.py" in diff_files[0].header - assert "file2.py" in diff_files[1].header - assert diff_files[0].hunks[0].additions == 1 - assert diff_files[0].hunks[0].deletions == 0 - assert diff_files[1].hunks[0].additions == 0 - assert diff_files[1].hunks[0].deletions == 1 - - def test_parse_multi_hunk_file(self): - """File with multiple hunks is parsed correctly.""" - content = """diff --git a/src/utils.py b/src/utils.py ---- a/src/utils.py -+++ b/src/utils.py -@@ -10,4 +10,5 @@ def helper(): - pass -+ # added comment - return True -@@ -50,3 +51,4 @@ def other(): - x = 1 -+ y = 2 - return x -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - - assert len(diff_files) == 1 - assert len(diff_files[0].hunks) == 2 - assert diff_files[0].total_additions == 2 - - def test_parse_new_file(self): - """New file diff is detected.""" - content = """diff --git a/newfile.py b/newfile.py -new file mode 100644 ---- /dev/null -+++ b/newfile.py -@@ -0,0 +1,3 @@ -+def new_func(): -+ pass -+ return None -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - - assert len(diff_files) == 1 - assert diff_files[0].is_new_file is True - assert diff_files[0].hunks[0].additions == 3 - - def test_parse_deleted_file(self): - """Deleted file diff is detected.""" - content = """diff --git a/oldfile.py b/oldfile.py -deleted file mode 100644 ---- a/oldfile.py -+++ /dev/null -@@ -1,2 +0,0 @@ --def old_func(): -- pass -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - - assert len(diff_files) == 1 - assert diff_files[0].is_deleted_file is True - assert diff_files[0].hunks[0].deletions == 2 - - def test_parse_binary_file(self): - """Binary file diff is detected.""" - content = """diff --git a/image.png b/image.png -Binary files a/image.png and b/image.png differ -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - - assert len(diff_files) == 1 - assert diff_files[0].is_binary is True - - class TestContextReduction: """Tests for context line reduction.""" @@ -336,58 +223,6 @@ class TestCompressionResult: assert result.tokens_saved_estimate == 900 -class TestHunkScoring: - """Tests for context-aware hunk scoring.""" - - def test_score_by_context_keywords(self): - """Hunks containing context keywords get higher scores.""" - content = """diff --git a/file.py b/file.py ---- a/file.py -+++ b/file.py -@@ -1,3 +1,4 @@ - normal context -+normal change - more context -@@ -10,3 +11,4 @@ - error handling -+fix the bug here - return result -""" - compressor = DiffCompressor() - _, diff_files = compressor._parse_diff(content.split("\n")) - compressor._score_hunks(diff_files, "fix error bug") - - # Second hunk should have higher score (contains "fix" and "bug") - assert len(diff_files[0].hunks) == 2 - assert diff_files[0].hunks[1].score > diff_files[0].hunks[0].score - - def test_score_priority_patterns(self): - """Hunks with priority patterns (error, security) score higher.""" - compressor = DiffCompressor() - - hunk_normal = DiffHunk( - header="@@ -1,1 +1,2 @@", - lines=["+normal change"], - additions=1, - ) - hunk_error = DiffHunk( - header="@@ -10,1 +10,2 @@", - lines=["+fix critical error"], - additions=1, - ) - - diff_file = DiffFile( - header="diff --git a/f.py b/f.py", - old_file="--- a/f.py", - new_file="+++ b/f.py", - hunks=[hunk_normal, hunk_error], - ) - - compressor._score_hunks([diff_file], "") - - assert hunk_error.score > hunk_normal.score - - class TestSmallDiffPassthrough: """Tests for small diff passthrough behavior.""" @@ -539,56 +374,6 @@ class TestEdgeCases: assert result.compressed is not None -class TestDiffHunkDataclass: - """Tests for DiffHunk dataclass.""" - - def test_change_count_property(self): - """change_count returns sum of additions and deletions.""" - hunk = DiffHunk( - header="@@ -1,5 +1,6 @@", - lines=["+a", "+b", "-c", " ctx"], - additions=2, - deletions=1, - ) - assert hunk.change_count == 3 - - def test_default_values(self): - """DiffHunk default values are correct.""" - hunk = DiffHunk(header="@@", lines=[]) - assert hunk.additions == 0 - assert hunk.deletions == 0 - assert hunk.context_lines == 0 - assert hunk.score == 0.0 - - -class TestDiffFileDataclass: - """Tests for DiffFile dataclass.""" - - def test_total_additions_property(self): - """total_additions sums across all hunks.""" - hunk1 = DiffHunk(header="@@", lines=[], additions=3) - hunk2 = DiffHunk(header="@@", lines=[], additions=5) - diff_file = DiffFile( - header="diff --git", - old_file="---", - new_file="+++", - hunks=[hunk1, hunk2], - ) - assert diff_file.total_additions == 8 - - def test_total_deletions_property(self): - """total_deletions sums across all hunks.""" - hunk1 = DiffHunk(header="@@", lines=[], deletions=2) - hunk2 = DiffHunk(header="@@", lines=[], deletions=4) - diff_file = DiffFile( - header="diff --git", - old_file="---", - new_file="+++", - hunks=[hunk1, hunk2], - ) - assert diff_file.total_deletions == 6 - - class TestConfigOptions: """Tests for configuration options.""" diff --git a/tests/test_transforms/test_diff_compressor_rust_parity.py b/tests/test_transforms/test_diff_compressor_rust_parity.py new file mode 100644 index 000000000..3feac188c --- /dev/null +++ b/tests/test_transforms/test_diff_compressor_rust_parity.py @@ -0,0 +1,119 @@ +"""Parity test: Rust-backed `DiffCompressor` vs recorded fixtures. + +Stage 3b verification, post-deletion. The Python implementation has +been retired; the only remaining `DiffCompressor` is Rust-backed via +PyO3 (`headroom._core`). These tests guard against regressions by +replaying every recorded fixture in +`tests/parity/fixtures/diff_compressor/` through the Rust backend and +asserting the output matches the recording byte-for-byte. + +Skipped automatically when the `headroom._core` wheel isn't installed +(e.g. CI lane without the maturin step). The Rust crate's own +`cargo run -p headroom-parity` covers the same fixtures from the Rust +side; this Python-side test catches PyO3 bridge regressions +(input/output mistranslation) that the Rust-only test cannot. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +def _has_core() -> bool: + try: + import headroom._core # noqa: F401 + + return True + except ImportError: + return False + + +pytestmark = pytest.mark.skipif( + not _has_core(), + reason="headroom._core wheel not installed (run `scripts/build_rust_extension.sh`)", +) + + +_FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "diff_compressor" + + +def _all_fixtures() -> list[Path]: + return sorted(_FIXTURES_DIR.glob("*.json")) + + +def test_at_least_27_fixtures_present(): + """Sanity check: the bug-fix and routing-gap fixtures landed.""" + fixtures = _all_fixtures() + assert len(fixtures) >= 27, ( + f"expected >= 27 fixtures, found {len(fixtures)}. " + "If you re-recorded and got fewer, something deleted them." + ) + + +@pytest.mark.parametrize("fixture_path", _all_fixtures(), ids=lambda p: p.name) +def test_rust_backend_matches_recorded_output(fixture_path: Path): + """Replay each recorded input through the Rust backend; every output + field must match the recording. Any mismatch is a PyO3 bridge bug or a + real Rust regression (cross-check with `cargo run -p headroom-parity` + to localize). + """ + from headroom.transforms.diff_compressor import DiffCompressor, DiffCompressorConfig + + rec = json.loads(fixture_path.read_text()) + cfg = DiffCompressorConfig(**rec["config"]) + result = DiffCompressor(cfg).compress(rec["input"]) + + expected = rec["output"] + # Field-by-field surfaces a single divergence rather than dumping the + # whole result on failure. + assert result.compressed == expected["compressed"], f"{fixture_path.name}: compressed differs" + assert result.original_line_count == expected["original_line_count"] + assert result.compressed_line_count == expected["compressed_line_count"] + assert result.files_affected == expected["files_affected"] + assert result.additions == expected["additions"] + assert result.deletions == expected["deletions"] + assert result.hunks_kept == expected["hunks_kept"] + assert result.hunks_removed == expected["hunks_removed"] + assert result.cache_key == expected["cache_key"] + + +def test_compress_with_stats_returns_python_dataclass_and_pyo3_stats(): + """The sidecar stats API returns a `(DiffCompressionResult, + _core.DiffCompressorStats)` tuple. Verify both halves are usable from + Python — PyO3 doesn't auto-promote the stats class to a dataclass. + """ + from headroom.transforms.diff_compressor import ( + DiffCompressionResult, + DiffCompressor, + DiffCompressorConfig, + ) + + diff = ( + "diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n@@ -1 +1 @@\n-old\n+new\n" + ) + "# pad\n" * 60 + result, stats = DiffCompressor(DiffCompressorConfig()).compress_with_stats(diff) + + assert isinstance(result, DiffCompressionResult) + assert stats.input_lines >= 60 + assert stats.processing_duration_us >= 0 + assert isinstance(stats.parse_warnings, list) + + +def test_content_router_uses_rust_backend(): + """Stage 3b deletion check: ContentRouter._get_diff_compressor must + return the (now Rust-only) DiffCompressor — there's no other backend + left, so this guards against an accidental re-introduction of a + Python-side stub or fallback. + """ + from headroom.transforms.content_router import ContentRouter, ContentRouterConfig + from headroom.transforms.diff_compressor import DiffCompressor + + router = ContentRouter(ContentRouterConfig()) + compressor = router._get_diff_compressor() + assert isinstance(compressor, DiffCompressor) + # The Rust delegation handle must be live — guards against a + # half-deleted shim that defines the class but not `_rust`. + assert hasattr(compressor, "_rust")