mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
83e27e5036
Adds crates/headroom-core/src/transforms/kompress.rs (672 lines): the Kompress ML prose compressor ported to Rust, running the ModernBERT tokenizer plus the kompress-v2-base ONNX model through ort with a cache-only loader that never touches the network. Parity-only. Nothing calls it: the only references outside the module are the pub mod / pub use declarations in transforms/mod.rs. live_zone.rs still carries TODO(PR-B4), so PlainText dispatch remains a no-op and Python continues to serve prose compression. The pyo3 bridge is untouched and no Python source changes, so the new engine is unreachable from the shipped package. #1155 wires it up. Ships 21 recorded parity fixtures, a KompressComparator in headroom-parity, and scripts/record_kompress_fixtures.py. Verified byte-identical to the recorded Python output: [kompress] total=21 matched=21 skipped=0 diffed=0 That required ONNX Runtime >= 1.24 (see #2591) — below it ort deadlocks instead of erroring, which is why these fixtures had never been run. In CI the model is absent from the HF cache, so the comparator errors and the fixtures report Skipped rather than hanging. Also gates the module behind the ml feature, matching magika_detector: kompress.rs uses ort, which is optional = true, so an unconditional pub mod broke cargo check --no-default-features (the static-musl path). CI does not catch that class of break because cargo test --workspace only builds default features.
47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Record standard parity fixtures for the Kompress transform only.
|
|
|
|
Drives the Python `KompressCompressor` (enable_ccr=False) over the shared
|
|
`_varied_kompress_inputs()` workload while `record_all()` has the compress
|
|
method patched, so only `tests/parity/fixtures/kompress/` is (re)written —
|
|
no churn to other transforms' fixtures.
|
|
|
|
Run after the model is cached:
|
|
python scripts/record_kompress_fixtures.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO))
|
|
|
|
|
|
def main() -> int:
|
|
from tests.parity.recorder import _varied_kompress_inputs, record_all
|
|
|
|
statuses = record_all()
|
|
if not statuses.get("kompress", "").startswith("patched"):
|
|
print(f"kompress not patched: {statuses.get('kompress')}", file=sys.stderr)
|
|
return 1
|
|
|
|
from headroom.transforms.kompress_compressor import (
|
|
KompressCompressor,
|
|
KompressConfig,
|
|
)
|
|
|
|
kc = KompressCompressor(KompressConfig(enable_ccr=False))
|
|
inputs = _varied_kompress_inputs()
|
|
for s in inputs:
|
|
kc.compress(s)
|
|
|
|
out_dir = REPO / "tests" / "parity" / "fixtures" / "kompress"
|
|
n = len(list(out_dir.glob("*.json")))
|
|
print(f"recorded {n} kompress fixtures from {len(inputs)} inputs -> {out_dir}", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|