From 5328d87b1eb70876d2df316180d3f7bdf7667fd7 Mon Sep 17 00:00:00 2001 From: chopratejas Date: Mon, 27 Apr 2026 00:22:23 -0700 Subject: [PATCH] feat(rust): pyo3 bridge for SmartCrusher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 3c.1b step 1: expose `SmartCrusherConfig`, `CrushResult`, and `SmartCrusher` to Python via `headroom._core`. The Python shim that delegates to it (replacing the 3669-line Python implementation) lands in the next commit; this commit just builds the bridge and a fixture-replay test that pins it. Surface: - `headroom._core.SmartCrusherConfig(**fields)` — every field of the Rust `SmartCrusherConfig` exposed as a kwarg with matching default. - `headroom._core.CrushResult` — read-only mirror of the Rust struct with `compressed`, `original`, `was_modified`, `strategy` getters. - `headroom._core.SmartCrusher(config=None)` — constructor accepts only `config`; the Python shim drops `relevance_config`, `scorer`, and `ccr_config` since Stage 3c.1 keeps those subsystems disabled. - `crush(content, query="", bias=1.0)` and `smart_crush_content(...)` methods mirror the Python signatures. Verification: - All 17 recorded parity fixtures byte-equal between Python and the PyO3 bridge (`tests/test_transforms/test_smart_crusher_rust_parity.py`, 18 tests pass — 1 fixture-count sanity + 17 fixtures). - The Rust-side `cargo run -p headroom-parity --bin parity-run -- run --only smart_crusher` was already 17/17 green. The two tests catch different regression classes: - Rust-only test: catches drift in the Rust port's logic. - Python bridge test: catches PyO3 input/output translation bugs. --- crates/headroom-py/src/lib.rs | 249 ++++++++++++++++++ .../test_smart_crusher_rust_parity.py | 87 ++++++ 2 files changed, 336 insertions(+) create mode 100644 tests/test_transforms/test_smart_crusher_rust_parity.py diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs index 0e51e1ba9..93d43983d 100644 --- a/crates/headroom-py/src/lib.rs +++ b/crates/headroom-py/src/lib.rs @@ -15,6 +15,10 @@ use std::collections::BTreeMap; +use headroom_core::transforms::smart_crusher::{ + CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher, + SmartCrusherConfig as RustSmartCrusherConfig, +}; use headroom_core::transforms::{ DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats, }; @@ -367,6 +371,248 @@ impl PyDiffCompressor { } } +// ─── SmartCrusherConfig ──────────────────────────────────────────────────── + +/// Mirror of `headroom.transforms.smart_crusher.SmartCrusherConfig`. +/// Defaults match Python's dataclass byte-for-byte. The constructor +/// accepts every field as a kwarg with the same name and type so the +/// Python shim can pass `SmartCrusherConfig(**asdict(py_cfg))`. +#[pyclass(name = "SmartCrusherConfig", module = "headroom._core")] +#[derive(Clone)] +struct PySmartCrusherConfig { + inner: RustSmartCrusherConfig, +} + +#[pymethods] +impl PySmartCrusherConfig { + #[new] + #[pyo3(signature = ( + enabled = true, + min_items_to_analyze = 5, + min_tokens_to_crush = 200, + variance_threshold = 2.0, + uniqueness_threshold = 0.1, + similarity_threshold = 0.8, + max_items_after_crush = 15, + preserve_change_points = true, + factor_out_constants = false, + include_summaries = false, + use_feedback_hints = true, + toin_confidence_threshold = 0.5, + dedup_identical_items = true, + first_fraction = 0.3, + last_fraction = 0.15, + relevance_threshold = 0.3, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + enabled: bool, + min_items_to_analyze: usize, + min_tokens_to_crush: usize, + variance_threshold: f64, + uniqueness_threshold: f64, + similarity_threshold: f64, + max_items_after_crush: usize, + preserve_change_points: bool, + factor_out_constants: bool, + include_summaries: bool, + use_feedback_hints: bool, + toin_confidence_threshold: f64, + dedup_identical_items: bool, + first_fraction: f64, + last_fraction: f64, + relevance_threshold: f64, + ) -> Self { + Self { + inner: RustSmartCrusherConfig { + enabled, + min_items_to_analyze, + min_tokens_to_crush, + variance_threshold, + uniqueness_threshold, + similarity_threshold, + max_items_after_crush, + preserve_change_points, + factor_out_constants, + include_summaries, + use_feedback_hints, + toin_confidence_threshold, + dedup_identical_items, + first_fraction, + last_fraction, + relevance_threshold, + }, + } + } + + #[getter] + fn enabled(&self) -> bool { + self.inner.enabled + } + #[getter] + fn min_items_to_analyze(&self) -> usize { + self.inner.min_items_to_analyze + } + #[getter] + fn min_tokens_to_crush(&self) -> usize { + self.inner.min_tokens_to_crush + } + #[getter] + fn variance_threshold(&self) -> f64 { + self.inner.variance_threshold + } + #[getter] + fn uniqueness_threshold(&self) -> f64 { + self.inner.uniqueness_threshold + } + #[getter] + fn similarity_threshold(&self) -> f64 { + self.inner.similarity_threshold + } + #[getter] + fn max_items_after_crush(&self) -> usize { + self.inner.max_items_after_crush + } + #[getter] + fn preserve_change_points(&self) -> bool { + self.inner.preserve_change_points + } + #[getter] + fn factor_out_constants(&self) -> bool { + self.inner.factor_out_constants + } + #[getter] + fn include_summaries(&self) -> bool { + self.inner.include_summaries + } + #[getter] + fn use_feedback_hints(&self) -> bool { + self.inner.use_feedback_hints + } + #[getter] + fn toin_confidence_threshold(&self) -> f64 { + self.inner.toin_confidence_threshold + } + #[getter] + fn dedup_identical_items(&self) -> bool { + self.inner.dedup_identical_items + } + #[getter] + fn first_fraction(&self) -> f64 { + self.inner.first_fraction + } + #[getter] + fn last_fraction(&self) -> f64 { + self.inner.last_fraction + } + #[getter] + fn relevance_threshold(&self) -> f64 { + self.inner.relevance_threshold + } + + fn __repr__(&self) -> String { + format!( + "SmartCrusherConfig(enabled={}, min_items_to_analyze={}, \ + min_tokens_to_crush={}, max_items_after_crush={}, \ + relevance_threshold={})", + self.inner.enabled, + self.inner.min_items_to_analyze, + self.inner.min_tokens_to_crush, + self.inner.max_items_after_crush, + self.inner.relevance_threshold, + ) + } +} + +// ─── CrushResult ─────────────────────────────────────────────────────────── + +/// Mirror of `headroom.transforms.smart_crusher.CrushResult`. Read-only; +/// the Python shim builds its own dataclass instance from these +/// attributes so callers that destructure with `asdict()` keep working. +#[pyclass(name = "CrushResult", module = "headroom._core")] +struct PyCrushResult { + inner: RustCrushResult, +} + +#[pymethods] +impl PyCrushResult { + #[getter] + fn compressed(&self) -> &str { + &self.inner.compressed + } + #[getter] + fn original(&self) -> &str { + &self.inner.original + } + #[getter] + fn was_modified(&self) -> bool { + self.inner.was_modified + } + #[getter] + fn strategy(&self) -> &str { + &self.inner.strategy + } + + fn __repr__(&self) -> String { + format!( + "CrushResult(compressed=<{} chars>, was_modified={}, strategy={:?})", + self.inner.compressed.len(), + self.inner.was_modified, + self.inner.strategy, + ) + } +} + +// ─── SmartCrusher ────────────────────────────────────────────────────────── + +/// Mirror of `headroom.transforms.smart_crusher.SmartCrusher`. +/// +/// Constructor accepts only `config` — Python's `relevance_config`, +/// `scorer`, and `ccr_config` parameters are handled in the Python +/// shim (Stage 3c.1 keeps the optional subsystems disabled in Rust; +/// the shim drops those args to preserve call-site compatibility). +#[pyclass(name = "SmartCrusher", module = "headroom._core")] +struct PySmartCrusher { + inner: RustSmartCrusher, +} + +#[pymethods] +impl PySmartCrusher { + #[new] + #[pyo3(signature = (config = None))] + fn new(config: Option<&PySmartCrusherConfig>) -> Self { + let cfg = config + .map(|c| c.inner.clone()) + .unwrap_or_default(); + Self { + inner: RustSmartCrusher::new(cfg), + } + } + + /// `crush(content, query="", bias=1.0) -> CrushResult`. Argument + /// order and keyword names mirror the Python implementation. + #[pyo3(signature = (content, query = "", bias = 1.0))] + fn crush(&self, content: &str, query: &str, bias: f64) -> PyCrushResult { + PyCrushResult { + inner: self.inner.crush(content, query, bias), + } + } + + /// `smart_crush_content(content, query="", bias=1.0) -> (str, bool, str)`. + /// Mirrors Python's `_smart_crush_content` — used by + /// `smart_crush_tool_output` convenience function and direct + /// callers that want the tuple form. + #[pyo3(signature = (content, query = "", bias = 1.0))] + fn smart_crush_content( + &self, + content: &str, + query: &str, + bias: f64, + ) -> (String, bool, String) { + self.inner.smart_crush_content(content, query, bias) + } +} + // ─── Module init ─────────────────────────────────────────────────────────── #[pymodule] @@ -376,5 +622,8 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/tests/test_transforms/test_smart_crusher_rust_parity.py b/tests/test_transforms/test_smart_crusher_rust_parity.py new file mode 100644 index 000000000..8e5a71276 --- /dev/null +++ b/tests/test_transforms/test_smart_crusher_rust_parity.py @@ -0,0 +1,87 @@ +"""Parity test: PyO3-backed `SmartCrusher` vs recorded fixtures. + +Stage 3c.1b verification — guards the PyO3 bridge against regressions +by replaying every recorded fixture in +`tests/parity/fixtures/smart_crusher/` through `headroom._core.SmartCrusher` +and asserting the output matches the recording byte-for-byte. + +Twin of `test_diff_compressor_rust_parity.py`. The Rust side runs the +same fixtures via `cargo run -p headroom-parity --bin parity-run -- +run --only smart_crusher`; this Python test specifically catches PyO3 +bridge regressions (input/output mistranslation) that the Rust-only +binary cannot. + +Skipped automatically when the `headroom._core` wheel isn't installed +(e.g. CI lane without the maturin step). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +def _has_core() -> bool: + try: + from headroom._core import SmartCrusher # noqa: F401 + + return True + except ImportError: + return False + + +pytestmark = pytest.mark.skipif( + not _has_core(), + reason="headroom._core wheel not installed (run `scripts/build_rust_extension.sh`)", +) + + +_FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "smart_crusher" + + +def _all_fixtures() -> list[Path]: + return sorted(_FIXTURES_DIR.glob("*.json")) + + +def test_at_least_17_fixtures_present(): + """Sanity check: the recorded fixture suite landed.""" + fixtures = _all_fixtures() + assert len(fixtures) >= 17, ( + f"expected >= 17 fixtures, found {len(fixtures)}. " + "If you re-recorded and got fewer, something deleted them." + ) + + +@pytest.mark.parametrize("fixture_path", _all_fixtures(), ids=lambda p: p.name) +def test_rust_backend_matches_recorded_output(fixture_path: Path): + """Replay each recorded input through the PyO3 bridge; every output + field must match the recording. Any mismatch is a bridge bug or a + Rust regression — cross-check with `cargo run -p headroom-parity`. + """ + from headroom._core import SmartCrusher, SmartCrusherConfig + + fixture = json.loads(fixture_path.read_text()) + inp = fixture["input"] + cfg_dict = fixture["config"] + expected = fixture["output"] + + cfg = SmartCrusherConfig(**cfg_dict) + crusher = SmartCrusher(cfg) + actual = crusher.crush(inp["content"], inp["query"], inp["bias"]) + + assert actual.compressed == expected["compressed"], ( + f"compressed bytes differ for {fixture_path.name}\n" + f" expected: {expected['compressed'][:120]!r}\n" + f" actual : {actual.compressed[:120]!r}" + ) + assert actual.original == expected["original"], f"original bytes differ for {fixture_path.name}" + assert actual.was_modified == expected["was_modified"], ( + f"was_modified differs for {fixture_path.name}: " + f"expected={expected['was_modified']} actual={actual.was_modified}" + ) + assert actual.strategy == expected["strategy"], ( + f"strategy differs for {fixture_path.name}: " + f"expected={expected['strategy']!r} actual={actual.strategy!r}" + )