diff --git a/crates/headroom-core/src/transforms/smart_crusher/config.rs b/crates/headroom-core/src/transforms/smart_crusher/config.rs index 8096b7624..69956e9e4 100644 --- a/crates/headroom-core/src/transforms/smart_crusher/config.rs +++ b/crates/headroom-core/src/transforms/smart_crusher/config.rs @@ -80,6 +80,13 @@ pub struct SmartCrusherConfig { /// still emit always; they have no Python equivalent and no /// production caller has asked for them to be suppressed. pub enable_ccr_marker: bool, + /// Strict lossless mode. When `true`, lossless tabular compaction + /// still applies, but any path that would otherwise need a CCR + /// marker — the lossy row-drop sentinel AND opaque-blob offload — + /// leaves the content uncompacted instead. The result is always + /// marker-free and byte-recoverable: rows are never dropped and + /// opaque cells render inline. Default `false` (markers allowed). + pub lossless_only: bool, /// Compaction heuristic: a field is "core" if it appears in at /// least this fraction of rows. Mirrors /// `CompactConfig::core_field_fraction`. Default 0.8. @@ -103,6 +110,22 @@ pub struct SmartCrusherConfig { pub compaction_max_buckets: usize, } +impl SmartCrusherConfig { + /// Whether opaque blobs should be offloaded to a `<>` marker. + /// + /// Single source of truth for the opaque-marker gate. Markers are + /// emitted only when CCR markers are enabled AND strict lossless + /// mode is off — `lossless_only` forbids any offload because the + /// marker would break the marker-free / byte-recoverable guarantee. + /// Both compaction-stage construction (`new` / + /// `with_compaction_format`) and the top-level `process_string` path + /// derive `ClassifyConfig::emit_opaque_markers` from this method so + /// the three call sites can never drift apart. + pub fn opaque_markers_enabled(&self) -> bool { + self.enable_ccr_marker && !self.lossless_only + } +} + impl Default for SmartCrusherConfig { fn default() -> Self { // These defaults must match smart_crusher.py:934-957 byte-for-byte. @@ -127,6 +150,7 @@ impl Default for SmartCrusherConfig { relevance_threshold: 0.3, lossless_min_savings_ratio: 0.15, enable_ccr_marker: true, + lossless_only: false, compaction_core_field_fraction: 0.8, compaction_heterogeneous_core_ratio: 0.6, compaction_max_flatten_inner_keys: 6, @@ -164,6 +188,7 @@ mod tests { assert_eq!(c.relevance_threshold, 0.3); assert_eq!(c.lossless_min_savings_ratio, 0.15); assert!(c.enable_ccr_marker); + assert!(!c.lossless_only); assert_eq!(c.compaction_core_field_fraction, 0.8); assert_eq!(c.compaction_heterogeneous_core_ratio, 0.6); assert_eq!(c.compaction_max_flatten_inner_keys, 6); diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs index 6af0fdc15..0c45f163a 100644 --- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs @@ -167,7 +167,7 @@ impl SmartCrusher { // the row-drop path), so `enable_ccr_marker=false` yields // marker-free, lossless output. Fixes #1091. classify: ClassifyConfig { - emit_opaque_markers: config.enable_ccr_marker, + emit_opaque_markers: config.opaque_markers_enabled(), ..ClassifyConfig::default() }, ..CompactConfig::default() @@ -202,7 +202,8 @@ impl SmartCrusher { /// `"markdown-kv"`). `None` for unknown names — callers own the /// fallback/error policy. `"csv-schema"` is equivalent to `new`. pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option { - let stage = CompactionStage::from_format_name(format_name)?; + let mut stage = CompactionStage::from_format_name(format_name)?; + stage.config.classify.emit_opaque_markers = config.opaque_markers_enabled(); Some( SmartCrusherBuilder::new(config) .with_default_oss_setup() @@ -605,7 +606,7 @@ impl SmartCrusher { // Gated by `enable_ccr_marker` so disabling markers stays lossless // here too (#1091). let cfg = ClassifyConfig { - emit_opaque_markers: self.config.enable_ccr_marker, + emit_opaque_markers: self.config.opaque_markers_enabled(), ..ClassifyConfig::default() }; if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) { @@ -698,6 +699,24 @@ impl SmartCrusher { } } + // ── Strict lossless-only mode ── + // + // The lossless attempt above either shipped or didn't. Either way + // `lossless_only` forbids the lossy row-drop fallback: dropping + // rows needs a CCR marker to stay recoverable, and the whole + // point of this mode is a marker-free, byte-recoverable result. + // Leave the array uncompacted instead. + if self.config.lossless_only { + return CrushArrayResult { + items: items.to_vec(), + strategy_info: "lossless_only:uncompacted".to_string(), + ccr_hash: None, + dropped_summary: String::new(), + compacted: None, + compaction_kind: None, + }; + } + // ── Lossy path: compress inline + cache full original via CCR ── // // The runtime caller (PyO3 bridge / proxy server) is expected @@ -705,6 +724,21 @@ impl SmartCrusher { // tool can serve dropped rows back to the LLM on demand. // **No data is lost** — "lossy" here means "compressed view // inline; full payload retrievable via CCR cache." + // + // Load-bearing invariant: a `lossless_only` crusher MUST NOT + // reach this point — the early return above guarantees it. The + // Python per-call override (`crush(..., lossless_only=True)`) + // relies on this: it swaps in a separate Rust crusher whose CCR + // store stays empty precisely because no lossless_only run ever + // executes the store write below. If that early return is ever + // removed, the alternate crusher's store would diverge and + // retrieval could resolve markers the prompt can't reference. + debug_assert!( + !self.config.lossless_only, + "lossy path reached under lossless_only — the early return \ + above must keep this codepath (and its CCR store write) \ + unreachable in strict lossless mode", + ); let effective_max_items = adaptive_k; let analysis = self.analyzer.analyze_array(items); @@ -1678,4 +1712,147 @@ mod tests { "default should write to ccr_store" ); } + + #[test] + fn enable_ccr_marker_false_suppresses_opaque_markers() { + // Opaque-blob path symmetry. A long string cell normally renders + // as a `<>` marker in the lossless table. + // With `enable_ccr_marker = false` it must render inline instead, + // so no configuration leaks markers into a "lossless-only" prompt. + let rows: Vec = (0..10) + .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)})) + .collect(); + + // ratio 0.0 forces the lossless table to ship, exercising the + // compactor's opaque arm directly (not the lossy row-drop path). + let off = SmartCrusher::new(SmartCrusherConfig { + lossless_min_savings_ratio: 0.0, + enable_ccr_marker: false, + ..SmartCrusherConfig::default() + }); + let rendered_off = off + .crush_array(&rows, "", 1.0) + .compacted + .expect("lossless table should ship at ratio 0.0"); + assert!( + !rendered_off.contains("< = (0..50) + .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)})) + .collect(); + + let crusher = SmartCrusher::new(SmartCrusherConfig { + lossless_min_savings_ratio: 0.99, // force the would-be-lossy path + lossless_only: true, + ..SmartCrusherConfig::default() + }); + let result = crusher.crush_array(&rows, "", 1.0); + + assert_eq!(result.items, rows, "lossless_only must not drop rows"); + assert!(result.ccr_hash.is_none(), "no hash under lossless_only"); + assert!( + result.dropped_summary.is_empty(), + "no drop sentinel under lossless_only: {:?}", + result.dropped_summary + ); + assert!( + result.compacted.is_none(), + "nothing shipped, nothing dropped" + ); + } + + #[test] + fn lossless_only_inlines_opaque_blobs_when_table_ships() { + // When the lossless table DOES win, opaque cells render inline + // (no marker) because lossless_only suppresses opaque offload. + let rows: Vec = (0..10) + .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)})) + .collect(); + let crusher = SmartCrusher::new(SmartCrusherConfig { + lossless_min_savings_ratio: 0.0, // table ships + lossless_only: true, + ..SmartCrusherConfig::default() + }); + let rendered = crusher + .crush_array(&rows, "", 1.0) + .compacted + .expect("table should ship at ratio 0.0"); + assert!( + !rendered.contains("< = Arc::new(InMemoryCcrStore::new()); + let cfg = SmartCrusherConfig { + lossless_min_savings_ratio: 0.99, // force the would-be-lossy path + lossless_only: true, + ..SmartCrusherConfig::default() + }; + let c = SmartCrusherBuilder::new(cfg) + .with_ccr_store(Arc::clone(&store)) + .build(); + let items: Vec = (0..50).map(|_| json!({"status": "ok"})).collect(); + + let store_len_before = store.len(); + let result = c.crush_array(&items, "", 1.0); + + assert_eq!( + result.items, items, + "lossless_only must keep every row (no drop)" + ); + assert!(result.ccr_hash.is_none(), "no hash under lossless_only"); + assert_eq!( + store.len(), + store_len_before, + "ccr_store grew under lossless_only — invariant violated" + ); + } } diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs index 97c902449..b528362e5 100644 --- a/crates/headroom-py/src/lib.rs +++ b/crates/headroom-py/src/lib.rs @@ -18,7 +18,9 @@ use std::collections::BTreeMap; use headroom_core::signals::{ ImportanceCategory, ImportanceContext, KeywordDetector, KeywordRegistry, LineImportanceDetector, }; -use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor; +use headroom_core::transforms::smart_crusher::compaction::{ + ClassifyConfig, CompactConfig, DocumentCompactor, +}; use headroom_core::transforms::smart_crusher::{ CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher, SmartCrusherConfig as RustSmartCrusherConfig, @@ -475,6 +477,7 @@ impl PySmartCrusherConfig { relevance_threshold = 0.3, lossless_min_savings_ratio = 0.15, enable_ccr_marker = true, + lossless_only = false, compaction_core_field_fraction = 0.8, compaction_heterogeneous_core_ratio = 0.6, compaction_max_flatten_inner_keys = 6, @@ -501,6 +504,7 @@ impl PySmartCrusherConfig { relevance_threshold: f64, lossless_min_savings_ratio: f64, enable_ccr_marker: bool, + lossless_only: bool, compaction_core_field_fraction: f64, compaction_heterogeneous_core_ratio: f64, compaction_max_flatten_inner_keys: usize, @@ -527,6 +531,7 @@ impl PySmartCrusherConfig { relevance_threshold, lossless_min_savings_ratio, enable_ccr_marker, + lossless_only, compaction_core_field_fraction, compaction_heterogeneous_core_ratio, compaction_max_flatten_inner_keys, @@ -605,6 +610,10 @@ impl PySmartCrusherConfig { self.inner.enable_ccr_marker } #[getter] + fn lossless_only(&self) -> bool { + self.inner.lossless_only + } + #[getter] fn lossless_min_savings_ratio(&self) -> f64 { self.inner.lossless_min_savings_ratio } @@ -854,7 +863,13 @@ impl PySmartCrusher { py.allow_threads(|| { let parsed: serde_json::Value = serde_json::from_str(&doc_json) .unwrap_or_else(|e| panic!("doc_json must be JSON: {e}")); - let mut dc = DocumentCompactor::new(); + let mut dc = DocumentCompactor::new().with_config(CompactConfig { + classify: ClassifyConfig { + emit_opaque_markers: self.inner.config.opaque_markers_enabled(), + ..ClassifyConfig::default() + }, + ..CompactConfig::default() + }); if let Some(store) = self.inner.ccr_store() { dc = dc.with_ccr_store(store.clone()); } diff --git a/headroom/config.py b/headroom/config.py index 0456f03b5..8dc3f35a8 100644 --- a/headroom/config.py +++ b/headroom/config.py @@ -428,6 +428,14 @@ class SmartCrusherConfig: # transforms-level dataclass. lossless_min_savings_ratio: float = 0.15 + # Strict lossless mode. When True, lossless tabular compaction still + # applies, but any path that would emit a CCR marker (lossy row-drop + # OR opaque-blob offload) leaves the content uncompacted instead, so + # the output is always marker-free and byte-recoverable. Mirrors the + # Rust default. See also `CCRConfig` — with this on, no `<>` + # markers are produced regardless of CCR settings. + lossless_only: bool = False + # Compaction heuristics (mirror Rust CompactConfig). A field is "core" # if present in at least this fraction of rows; arrays whose key sets # are mostly non-core are bucketed by a discriminator instead. diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index f5dea7796..26d79b869 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -632,6 +632,15 @@ class HeadroomProxy( # to PASSTHROUGH instead of the default KOMPRESS fallback strategy. if config.disable_kompress_fallback: router_config.fallback_strategy = CompressionStrategy.PASSTHROUGH + # `HEADROOM_LOSSLESS_ONLY=1` routes SmartCrusher through strict + # marker-free mode: lossless tabular compaction still applies, but + # any path that would emit a `<>` marker (row-drop or + # opaque-blob offload) leaves the content uncompacted instead — so + # the session needs no CCR retrieval round-trips to stay recoverable. + if "HEADROOM_LOSSLESS_ONLY" in os.environ: + router_config.smart_crusher_lossless_only = _get_env_bool( + "HEADROOM_LOSSLESS_ONLY", False + ) # A non-None exclude_tools replaces DEFAULT_EXCLUDE_TOOLS in # ContentRouter, so merge rather than assign. if config.exclude_tools: diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py index 702f0a1c2..9386e0497 100644 --- a/headroom/transforms/content_router.py +++ b/headroom/transforms/content_router.py @@ -689,6 +689,11 @@ class ContentRouterConfig: ccr_inject_marker: bool = True # Add retrieval markers to compressed content smart_crusher_max_items_after_crush: int | None = None smart_crusher_with_compaction: bool = True + # Strict lossless-only mode for SmartCrusher. None → leave the + # crusher config's own value untouched; True/False force it. Wired + # from the proxy's `HEADROOM_LOSSLESS_ONLY` env var so a real session + # can run marker-free without constructing the crusher by hand. + smart_crusher_lossless_only: bool | None = None # Tag protection: preserve custom/workflow XML tags from text compression. # When False (default), entire content blocks are @@ -1872,6 +1877,8 @@ class ContentRouter(Transform): crusher_config.max_items_after_crush = ( self.config.smart_crusher_max_items_after_crush ) + if self.config.smart_crusher_lossless_only is not None: + crusher_config.lossless_only = self.config.smart_crusher_lossless_only self._smart_crusher = SmartCrusher( config=crusher_config, ccr_config=ccr_config, diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py index fe283e177..9823040e4 100644 --- a/headroom/transforms/smart_crusher.py +++ b/headroom/transforms/smart_crusher.py @@ -187,6 +187,13 @@ class SmartCrusherConfig: # and KV experiments — KV repeats field names per row, so it clears # the gate less often than CSV. lossless_min_savings_ratio: float = 0.15 + # Strict lossless mode. When True, lossless tabular compaction still + # applies, but any path that would otherwise emit a CCR marker — the + # lossy row-drop sentinel AND opaque-blob offload — leaves the content + # uncompacted instead. The output is always marker-free and fully + # byte-recoverable: rows are never dropped and opaque cells render + # inline. Default False (markers allowed). Mirrors the Rust default. + lossless_only: bool = False # Compaction heuristics (mirror Rust CompactConfig; see # crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs). @@ -227,6 +234,7 @@ class SmartCrusher(Transform): with_compaction: bool = True, observer: Any = None, compaction_format: str | None = None, + lossless_only: bool | None = None, ): # Hard import — no Python fallback. If the wheel is missing the # caller must build it (scripts/build_rust_extension.sh) or @@ -242,6 +250,16 @@ class SmartCrusher(Transform): cfg = config or SmartCrusherConfig() self.config = cfg self._with_compaction = with_compaction + # Strict lossless mode. An explicit `lossless_only=` kwarg wins + # over the config field, so callers can flip it without rebuilding + # a whole config. `crush(..., lossless_only=...)` overrides again + # per call. getattr fallback: callers may pass the SDK-side + # `headroom.config.SmartCrusherConfig`, which also carries it. + self._lossless_only = ( + bool(getattr(cfg, "lossless_only", False)) + if lossless_only is None + else bool(lossless_only) + ) # `observer`: see `headroom.transforms.observability`. The # legacy proxy pipeline uses SmartCrusher.apply() directly # (no ContentRouter); without an observer here, those @@ -315,39 +333,46 @@ class SmartCrusher(Transform): # Build the Rust crusher with every field from the Python # config, plus the relevance_threshold default (0.3) — the # Python dataclass doesn't carry that field; it lives on - # `RelevanceScorerConfig` instead. - rust_cfg = _RustSmartCrusherConfig( - enabled=cfg.enabled, - min_items_to_analyze=cfg.min_items_to_analyze, - min_tokens_to_crush=cfg.min_tokens_to_crush, - variance_threshold=cfg.variance_threshold, - uniqueness_threshold=cfg.uniqueness_threshold, - similarity_threshold=cfg.similarity_threshold, - max_items_after_crush=cfg.max_items_after_crush, - preserve_change_points=cfg.preserve_change_points, - factor_out_constants=cfg.factor_out_constants, - include_summaries=cfg.include_summaries, - use_feedback_hints=cfg.use_feedback_hints, - toin_confidence_threshold=cfg.toin_confidence_threshold, - dedup_identical_items=cfg.dedup_identical_items, - first_fraction=cfg.first_fraction, - last_fraction=cfg.last_fraction, - relevance_threshold=0.3, - enable_ccr_marker=( + # `RelevanceScorerConfig` instead. Kept as a kwargs dict so the + # per-call `crush(..., lossless_only=...)` override can rebuild an + # alternate crusher with just that one field flipped. + self._RustSmartCrusher = _RustSmartCrusher + self._RustSmartCrusherConfig = _RustSmartCrusherConfig + self._rust_cfg_kwargs = { + "enabled": cfg.enabled, + "min_items_to_analyze": cfg.min_items_to_analyze, + "min_tokens_to_crush": cfg.min_tokens_to_crush, + "variance_threshold": cfg.variance_threshold, + "uniqueness_threshold": cfg.uniqueness_threshold, + "similarity_threshold": cfg.similarity_threshold, + "max_items_after_crush": cfg.max_items_after_crush, + "preserve_change_points": cfg.preserve_change_points, + "factor_out_constants": cfg.factor_out_constants, + "include_summaries": cfg.include_summaries, + "use_feedback_hints": cfg.use_feedback_hints, + "toin_confidence_threshold": cfg.toin_confidence_threshold, + "dedup_identical_items": cfg.dedup_identical_items, + "first_fraction": cfg.first_fraction, + "last_fraction": cfg.last_fraction, + "relevance_threshold": 0.3, + "enable_ccr_marker": ( self._ccr_config.enabled and self._ccr_config.inject_retrieval_marker ), + "lossless_only": self._lossless_only, # getattr fallbacks: callers may pass the structurally-similar # `headroom.config.SmartCrusherConfig` (MCP server, SDK) or a # pre-existing config object that predates these fields. - lossless_min_savings_ratio=getattr(cfg, "lossless_min_savings_ratio", 0.15), - compaction_core_field_fraction=getattr(cfg, "compaction_core_field_fraction", 0.8), - compaction_heterogeneous_core_ratio=getattr( + "lossless_min_savings_ratio": getattr(cfg, "lossless_min_savings_ratio", 0.15), + "compaction_core_field_fraction": getattr(cfg, "compaction_core_field_fraction", 0.8), + "compaction_heterogeneous_core_ratio": getattr( cfg, "compaction_heterogeneous_core_ratio", 0.6 ), - compaction_max_flatten_inner_keys=getattr(cfg, "compaction_max_flatten_inner_keys", 6), - compaction_min_buckets=getattr(cfg, "compaction_min_buckets", 2), - compaction_max_buckets=getattr(cfg, "compaction_max_buckets", 8), - ) + "compaction_max_flatten_inner_keys": getattr( + cfg, "compaction_max_flatten_inner_keys", 6 + ), + "compaction_min_buckets": getattr(cfg, "compaction_min_buckets", 2), + "compaction_max_buckets": getattr(cfg, "compaction_max_buckets", 8), + } # Default: lossless-first compaction (PR4). Lossless wins for # cleanly tabular input where it saves ≥ 30% bytes; otherwise # falls through to the lossy path with CCR-Dropped retrieval @@ -373,24 +398,59 @@ class SmartCrusher(Transform): f"expected one of: {', '.join(_SUPPORTED_COMPACTION_FORMATS)}" ) self._compaction_format = resolved_format if with_compaction else None - if not with_compaction: - self._rust = _RustSmartCrusher.without_compaction(rust_cfg) - elif resolved_format == "csv-schema": - # Keep the `new()` constructor for the default path so its - # byte-parity coverage stays on the exact production - # codepath. - self._rust = _RustSmartCrusher(rust_cfg) - else: - self._rust = _RustSmartCrusher.with_compaction_format(rust_cfg, resolved_format) + self._resolved_compaction_format = resolved_format + # Cache of Rust crushers keyed by lossless_only, so a per-call + # override builds the alternate at most once. + self._rust_by_lossless_only: dict[bool, Any] = {} + self._rust = self._build_rust(self._lossless_only) - def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult: + def _build_rust(self, lossless_only: bool) -> Any: + """Build (and cache) the Rust crusher for a `lossless_only` value.""" + cached = self._rust_by_lossless_only.get(lossless_only) + if cached is not None: + return cached + kwargs = dict(self._rust_cfg_kwargs) + kwargs["lossless_only"] = lossless_only + rust_cfg = self._RustSmartCrusherConfig(**kwargs) + if not self._with_compaction: + rust = self._RustSmartCrusher.without_compaction(rust_cfg) + elif self._resolved_compaction_format == "csv-schema": + # Keep the `new()` constructor for the default path so its + # byte-parity coverage stays on the exact production codepath. + rust = self._RustSmartCrusher(rust_cfg) + else: + rust = self._RustSmartCrusher.with_compaction_format( + rust_cfg, self._resolved_compaction_format + ) + self._rust_by_lossless_only[lossless_only] = rust + return rust + + def crush( + self, + content: str, + query: str = "", + bias: float = 1.0, + lossless_only: bool | None = None, + ) -> CrushResult: """Crush a single JSON content string. Mirrors the retired Python method. Returns a `CrushResult` dataclass so call sites that destructure with `asdict()` keep working. + + `lossless_only` overrides the configured strict-lossless mode for + this call only. When `True`, the output is guaranteed marker-free + and byte-recoverable: lossless tabular compaction still applies, + but any path that would need a CCR marker (row-drop or + opaque-blob offload) leaves the content uncompacted instead. + `None` (default) uses the instance's configured value. """ - r = self._rust.crush(content, query, bias) + rust = ( + self._rust + if lossless_only is None or bool(lossless_only) == self._lossless_only + else self._build_rust(bool(lossless_only)) + ) + r = rust.crush(content, query, bias) # Re-attach the TOIN learning loop. The retired Python class # recorded compressions into TOIN inline; the Rust port doesn't # know about TOIN, and `ContentRouter._record_to_toin` skips @@ -1017,6 +1077,7 @@ def smart_crush_tool_output( config: SmartCrusherConfig | None = None, ccr_config: CCRConfig | None = None, with_compaction: bool = True, + lossless_only: bool | None = None, ) -> tuple[str, bool, str]: """Compress a single tool output. Returns `(crushed, was_modified, info)`. @@ -1024,6 +1085,14 @@ def smart_crush_tool_output( Defaults to the PR4 lossless-first behavior; pass `with_compaction=False` to exercise the legacy lossy-only path (still useful for retention-property tests). + + `lossless_only=True` forces strict lossless mode: the output is + marker-free and byte-recoverable (no row drops, opaque blobs inline). """ - crusher = SmartCrusher(config=config, ccr_config=ccr_config, with_compaction=with_compaction) + crusher = SmartCrusher( + config=config, + ccr_config=ccr_config, + with_compaction=with_compaction, + lossless_only=lossless_only, + ) return crusher._smart_crush_content(content) diff --git a/tests/test_agent_savings.py b/tests/test_agent_savings.py index 38dbe2b4c..1eeae1519 100644 --- a/tests/test_agent_savings.py +++ b/tests/test_agent_savings.py @@ -358,6 +358,29 @@ def test_agent_90_router_uses_ccr_sampling_not_lossless_table() -> None: assert crusher._with_compaction is False +def test_router_lossless_only_flag_reaches_crusher() -> None: + # HEADROOM_LOSSLESS_ONLY=1 sets this field on the proxy router; it + # must flow through to the SmartCrusher so a real proxy session runs + # strict marker-free mode. + router = ContentRouter(ContentRouterConfig(smart_crusher_lossless_only=True)) + + crusher = router._get_smart_crusher() + + assert crusher is not None + assert crusher._lossless_only is True + + +def test_router_lossless_only_defaults_off() -> None: + # Unset (None) must not force the flag — default crushers stay in + # the marker-emitting mode. + router = ContentRouter(ContentRouterConfig()) + + crusher = router._get_smart_crusher() + + assert crusher is not None + assert crusher._lossless_only is False + + def test_agent_90_router_json_tool_output_reaches_target_with_needle() -> None: needle = "CRITICAL_NEEDLE_42" rows = [ diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.py index 3e7aa22e5..616806d99 100644 --- a/tests/test_transforms/test_smart_crusher_bugs.py +++ b/tests/test_transforms/test_smart_crusher_bugs.py @@ -124,6 +124,45 @@ class TestRecursionDepthLimit: assert isinstance(parsed, list) +class TestLosslessOnlyMode: + """`lossless_only` produces marker-free, byte-recoverable output. + + Strict mode: lossless tabular compaction still applies, but any path + that would need a CCR marker (lossy row-drop OR opaque-blob offload) + leaves the content uncompacted instead — so the result is always + marker-free and decodes back to the original input without loss. + """ + + def _droppable_rows(self) -> list[dict]: + return [{"path": "a.py", "line": i, "content": "x" * 300} for i in range(50)] + + def test_lossless_only_is_marker_free_and_byte_recoverable(self) -> None: + rows = self._droppable_rows() + config = SmartCrusherConfig( + min_items_to_analyze=3, + min_tokens_to_crush=0, + lossless_min_savings_ratio=0.99, # force the would-be-lossy path + lossless_only=True, + ) + out = SmartCrusher(config=config).crush(json.dumps(rows)) + assert "< None: + # Configured non-strict, but the per-call kwarg forces strict mode + # for this call: marker-free and fully recoverable. + rows = self._droppable_rows() + config = SmartCrusherConfig( + min_items_to_analyze=3, + min_tokens_to_crush=0, + lossless_min_savings_ratio=0.99, + ) + crusher = SmartCrusher(config=config) + out = crusher.crush(json.dumps(rows), lossless_only=True) + assert "<