From d219beecab0124bcd6392e246907021d631e7710 Mon Sep 17 00:00:00 2001 From: chopratejas Date: Sun, 26 Apr 2026 16:45:42 -0700 Subject: [PATCH] feat(rust): scaffold smart_crusher module + foundational helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 3c.1 — like-for-like Rust port of `headroom/transforms/smart_crusher.py`. This commit lays the foundation: module layout, configuration, foundational data types, and the simpler helpers (classification, hashing, anchors, basic statistics). Subsequent commits add the analyzer, crushers, plan execution, and the orchestrator. # What's in this commit `crates/headroom-core/src/transforms/smart_crusher/`: - `mod.rs` — module entry, public re-exports, port narrative. - `classifier.rs` — `classify_array` / `ArrayType` (dict/string/number/ bool/nested/mixed/empty). Direct port of `_classify_array`. - `config.rs` — `SmartCrusherConfig` with defaults pinned to Python byte-for-byte. - `hashing.rs` — `hash_field_name` (SHA-256 truncated to 16 hex chars), matches `hashlib.sha256(name.encode()).hexdigest()[:16]` exactly. - `statistics.rs` — `is_uuid_format`, `calculate_string_entropy`, `detect_sequential_pattern` (with **BUG #2 fix** — see below). - `anchors.rs` — `extract_query_anchors`, `item_matches_anchors`. Five regex patterns ported via `std::sync::LazyLock`. - `types.rs` — `CompressionStrategy`, `FieldStats`, `CrushabilityAnalysis`, `ArrayAnalysis`, `CompressionPlan`, `CrushResult`. Field-by-field mirror of the Python @dataclasses so the PyO3 bridge in 3c.1b can reconstruct them without manual translators. # Bug #2 fixed in this commit (Python fix lands later in same PR) `smart_crusher.py:444-448` — `_detect_sequential_pattern` calls `int(string_value)` and silently strips zero-padding, so padded string IDs like `["001", "002", ..., "100"]` get misclassified as a sequential numeric pattern. Fix: track whether each parsed numeric value originated as a string. If EVERY parsed value was a string, refuse to flag as sequential. Mixed numeric+string fields still detect correctly because the unambiguous numerics dominate. Test: `bug2_zero_padded_strings_no_longer_misclassified`. # What's NOT in this commit (subsequent commits) - `SmartAnalyzer` — `analyze_array`, `_analyze_field`, `_detect_change_points`, `_detect_pattern`, `_detect_temporal_field`, `analyze_crushability`, `_select_strategy`, `_estimate_reduction`. - The five array crushers (`_crush_array`, `_crush_string_array`, `_crush_number_array`, `_crush_mixed_array`, `_crush_object`). - Planning (`_compute_k_split`, `_create_plan`, `_plan_*` family). - Orchestration (`_prioritize_indices`, `_deduplicate_indices_by_content`, `_fill_remaining_slots`). - `SmartCrusher` orchestrator class itself. - Parity harness fixtures. - The remaining 3 Python bug fixes (#1, #3, #4) — landed alongside the code paths they affect. # Build / test - `cargo build -p headroom-core` — clean. - `cargo clippy -p headroom-core -- -D warnings` — clean. - 55 new unit tests across the 6 new files, all passing. Architectural improvements (lossless-first, unified saliency score, structured CCR markers) are deferred to Stage 3c.2 — see design doc at `~/Desktop/SmartCrusher-Architecture-Improvements.md`. --- .claude-plugin/marketplace.json | 4 +- .github/plugin/marketplace.json | 4 +- Cargo.lock | 12 + crates/headroom-core/Cargo.toml | 3 + crates/headroom-core/src/transforms/mod.rs | 1 + .../src/transforms/smart_crusher/anchors.rs | 251 ++++++++++++ .../transforms/smart_crusher/classifier.rs | 203 ++++++++++ .../src/transforms/smart_crusher/config.rs | 99 +++++ .../src/transforms/smart_crusher/hashing.rs | 58 +++ .../src/transforms/smart_crusher/mod.rs | 50 +++ .../transforms/smart_crusher/statistics.rs | 375 ++++++++++++++++++ .../src/transforms/smart_crusher/types.rs | 244 ++++++++++++ .../.claude-plugin/plugin.json | 2 +- .../.github/plugin/plugin.json | 2 +- 14 files changed, 1302 insertions(+), 6 deletions(-) create mode 100644 crates/headroom-core/src/transforms/smart_crusher/anchors.rs create mode 100644 crates/headroom-core/src/transforms/smart_crusher/classifier.rs create mode 100644 crates/headroom-core/src/transforms/smart_crusher/config.rs create mode 100644 crates/headroom-core/src/transforms/smart_crusher/hashing.rs create mode 100644 crates/headroom-core/src/transforms/smart_crusher/mod.rs create mode 100644 crates/headroom-core/src/transforms/smart_crusher/statistics.rs create mode 100644 crates/headroom-core/src/transforms/smart_crusher/types.rs diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 78381697b..579b17788 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.", - "version": "0.11.0" + "version": "0.10.17" }, "plugins": [ { "name": "headroom", "source": "./plugins/headroom-agent-hooks", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", - "version": "0.11.0", + "version": "0.10.17", "author": { "name": "Headroom Contributors", "url": "https://github.com/chopratejas/headroom" diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 78381697b..579b17788 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.", - "version": "0.11.0" + "version": "0.10.17" }, "plugins": [ { "name": "headroom", "source": "./plugins/headroom-agent-hooks", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", - "version": "0.11.0", + "version": "0.10.17", "author": { "name": "Headroom Contributors", "url": "https://github.com/chopratejas/headroom" diff --git a/Cargo.lock b/Cargo.lock index cbf163166..4d52bfd86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -933,6 +933,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "thiserror 1.0.69", "tiktoken-rs", "tokenizers", @@ -2238,6 +2239,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml index 929230211..d522455d7 100644 --- a/crates/headroom-core/Cargo.toml +++ b/crates/headroom-core/Cargo.toml @@ -26,6 +26,9 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq", "rustl # `md5` for the CCR cache_key. Python's compression_store hashes the original # diff with MD5 truncated to 24 hex chars; we must match byte-for-byte. md-5 = "0.10" +# `sha2` for `_hash_field_name` in smart_crusher (SHA256 truncated to 16 +# hex chars). Python uses `hashlib.sha256` so we need byte-exact parity. +sha2 = "0.10" # `regex` is already a transitive dep of tokenizers; depend on it directly so # our hunk-header parser and priority-pattern matcher have a stable surface. regex = "1" diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rs index f0fc4bc49..abaf33363 100644 --- a/crates/headroom-core/src/transforms/mod.rs +++ b/crates/headroom-core/src/transforms/mod.rs @@ -16,6 +16,7 @@ //! prod and are returned alongside the parity-equal output for tests. pub mod diff_compressor; +pub mod smart_crusher; pub use diff_compressor::{ DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats, diff --git a/crates/headroom-core/src/transforms/smart_crusher/anchors.rs b/crates/headroom-core/src/transforms/smart_crusher/anchors.rs new file mode 100644 index 000000000..6a6f55eae --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/anchors.rs @@ -0,0 +1,251 @@ +//! Legacy regex-based query anchor extraction. +//! +//! Direct port of `extract_query_anchors` and `item_matches_anchors` +//! (`smart_crusher.py:99-168`). The Python doc-comment marks both as +//! DEPRECATED in favor of `RelevanceScorer`, but they're still called +//! by the live SmartCrusher path on every invocation, so we port them +//! faithfully. +//! +//! # Why regex parity matters +//! +//! These regexes drive which array items survive compression. A subtle +//! difference between Python's `re` engine and Rust's `regex` crate +//! (e.g. word-boundary behavior on Unicode, or repetition greediness) +//! would silently change which anchors are detected and which items +//! survive. The patterns below are pinned to lowercase ASCII inputs +//! and use only ASCII-safe constructs to keep behavior identical. + +use regex::Regex; +use serde_json::Value; +use std::collections::HashSet; +use std::sync::LazyLock; + +// --------------------------------------------------------------- +// Pattern definitions — direct ports of the module-level Python regexes +// at `smart_crusher.py:85-93`. `std::sync::LazyLock` (stable since Rust +// 1.80) is the modern equivalent of `once_cell::sync::Lazy`, mirroring +// Python's `re.compile` at module import time. +// --------------------------------------------------------------- + +/// `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b` +static UUID_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b") + .expect("UUID_PATTERN") +}); + +/// 4+ digit numbers (likely IDs). Python: `r"\b\d{4,}\b"`. +static NUMERIC_ID_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"\b\d{4,}\b").expect("NUMERIC_ID_PATTERN")); + +/// Hostname pattern. Matches `host.tld` with optional `.tld2`. Python: +/// `r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b"`. +static HOSTNAME_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b") + .expect("HOSTNAME_PATTERN") +}); + +/// Short quoted strings (single OR double quotes), 1-50 chars between +/// quotes. Python: `r"['\"]([^'\"]{1,50})['\"]"`. +static QUOTED_STRING_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r#"['"]([^'"]{1,50})['"]"#).expect("QUOTED_STRING_PATTERN")); + +/// Email addresses. Python: `r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"`. +/// (Note Python's `[A-Z|a-z]` includes a literal `|` in the character +/// class — almost certainly a typo, but we faithfully port it for +/// parity. Real-world impact is nil since `|` doesn't appear in TLDs.) +static EMAIL_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b").expect("EMAIL_PATTERN") +}); + +/// Hostname false-positive blocklist. Python uses a set literal at +/// `smart_crusher.py:137`. We mirror exactly — these strings get +/// dropped from anchor results. +const HOSTNAME_FALSE_POSITIVES: &[&str] = &["e.g", "i.e", "etc."]; + +/// Extract query anchors from user text. **DEPRECATED** in Python in +/// favor of `RelevanceScorer`, but still called by the live path — +/// ported as-is. +/// +/// Output is a set of lowercased anchor strings. Order is not +/// significant (Python returns `set[str]`). +pub fn extract_query_anchors(text: &str) -> HashSet { + let mut anchors = HashSet::new(); + + if text.is_empty() { + return anchors; + } + + // UUIDs — lowercase the match. + for m in UUID_PATTERN.find_iter(text) { + anchors.insert(m.as_str().to_lowercase()); + } + + // Numeric IDs — Python keeps original case (digits, no transform needed). + for m in NUMERIC_ID_PATTERN.find_iter(text) { + anchors.insert(m.as_str().to_string()); + } + + // Hostnames — lowercase, filter false positives. + for m in HOSTNAME_PATTERN.find_iter(text) { + let lc = m.as_str().to_lowercase(); + if !HOSTNAME_FALSE_POSITIVES.contains(&lc.as_str()) { + anchors.insert(lc); + } + } + + // Quoted strings — capture group 1 (the content between quotes), + // require trim().len() >= 2 (Python's `if len(match.strip()) >= 2`). + for caps in QUOTED_STRING_PATTERN.captures_iter(text) { + if let Some(inner) = caps.get(1) { + if inner.as_str().trim().len() >= 2 { + anchors.insert(inner.as_str().to_lowercase()); + } + } + } + + // Emails — lowercase. + for m in EMAIL_PATTERN.find_iter(text) { + anchors.insert(m.as_str().to_lowercase()); + } + + anchors +} + +/// Check if a JSON object matches any query anchors. +/// +/// Direct port of `item_matches_anchors` (Python `smart_crusher.py:152-168`). +/// Python uses `str(item).lower()` which produces Python's `dict.__str__` +/// representation. We mirror by serializing with `serde_json` and +/// lowercasing — this isn't byte-identical to Python's `str(dict)` +/// (Python uses single quotes, JSON uses double; Python's bool is +/// `True`/`False`, JSON's is `true`/`false`), so for cross-language +/// parity we need a string form that matches Python's. We document this +/// gap and fix it in the analyzer integration. +/// +/// **WARNING:** `str(item).lower()` in Python produces: +/// `{'key': 'value', 'count': 5, 'ok': True}` +/// while `serde_json::to_string(&item)` produces: +/// `{"key":"value","count":5,"ok":true}` +/// +/// The anchor matching is substring-based (`anchor in item_str`), so +/// this difference matters: if an anchor is `"true"` it matches the +/// JSON form but not the Python form, and vice versa for `"True"`. +/// +/// **Resolution:** when items reach the matcher they're already +/// lowercased, so `True` → `true` after `.lower()`, removing one source +/// of drift. The remaining drift (single vs double quotes, trailing +/// whitespace) is unlikely to affect anchor matching in practice. We +/// pin behavior with fixtures and move on. +pub fn item_matches_anchors(item: &Value, anchors: &HashSet) -> bool { + if anchors.is_empty() { + return false; + } + + // Python: `str(item).lower()`. We approximate via JSON serialization + // followed by `.lower()` — see WARNING above for the gap. + let item_str = match serde_json::to_string(item) { + Ok(s) => s.to_lowercase(), + Err(_) => return false, + }; + + anchors.iter().any(|a| item_str.contains(a)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn empty_text_no_anchors() { + assert!(extract_query_anchors("").is_empty()); + } + + #[test] + fn extracts_uuid_lowercased() { + let anchors = extract_query_anchors("see id 550E8400-E29B-41D4-A716-446655440000 plz"); + assert!(anchors.contains("550e8400-e29b-41d4-a716-446655440000")); + } + + #[test] + fn extracts_numeric_id_unchanged() { + let anchors = extract_query_anchors("user 12345 reported issue"); + assert!(anchors.contains("12345")); + } + + #[test] + fn three_digit_number_not_anchor() { + // Pattern requires 4+ digits. + let anchors = extract_query_anchors("user 123 reported issue"); + assert!(!anchors.iter().any(|a| a == "123")); + } + + #[test] + fn extracts_hostname() { + let anchors = extract_query_anchors("connect to api.example.com asap"); + assert!(anchors.contains("api.example.com")); + } + + #[test] + fn hostname_false_positive_filtered() { + // "e.g" is in the blocklist — must NOT appear as an anchor even + // though it matches the regex. + let anchors = extract_query_anchors("test e.g.com endpoint"); + // "e.g" is filtered, but "e.g.com" or other longer matches may + // pass; we only assert "e.g" itself is gone. + assert!(!anchors.contains("e.g")); + } + + #[test] + fn extracts_quoted_string_double() { + let anchors = extract_query_anchors(r#"find the "user_name" field"#); + assert!(anchors.contains("user_name")); + } + + #[test] + fn extracts_quoted_string_single() { + let anchors = extract_query_anchors("find the 'user_name' field"); + assert!(anchors.contains("user_name")); + } + + #[test] + fn very_short_quoted_skipped() { + // Less than 2 chars after trim — skipped. + let anchors = extract_query_anchors(r#"the "x" thing"#); + assert!(!anchors.contains("x")); + } + + #[test] + fn extracts_email() { + let anchors = extract_query_anchors("contact USER@example.COM please"); + assert!(anchors.contains("user@example.com")); + } + + #[test] + fn item_matches_anchors_empty_set() { + let empty = HashSet::new(); + assert!(!item_matches_anchors(&json!({"a": 1}), &empty)); + } + + #[test] + fn item_matches_anchor_in_value() { + let anchors: HashSet = ["alice".to_string()].into_iter().collect(); + assert!(item_matches_anchors(&json!({"name": "Alice"}), &anchors)); + } + + #[test] + fn item_matches_anchor_in_key() { + let anchors: HashSet = ["status".to_string()].into_iter().collect(); + // The anchor "status" appears in the JSON-serialized key. + assert!(item_matches_anchors( + &json!({"status": "ok"}), + &anchors + )); + } + + #[test] + fn item_no_match_with_unrelated_anchor() { + let anchors: HashSet = ["xyz123".to_string()].into_iter().collect(); + assert!(!item_matches_anchors(&json!({"a": "b"}), &anchors)); + } +} diff --git a/crates/headroom-core/src/transforms/smart_crusher/classifier.rs b/crates/headroom-core/src/transforms/smart_crusher/classifier.rs new file mode 100644 index 000000000..d8a04892d --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/classifier.rs @@ -0,0 +1,203 @@ +//! JSON array element-type classification. +//! +//! Direct port of `_classify_array` (Python `smart_crusher.py:341-368`). +//! Classification drives compression strategy: dict arrays go through +//! `_crush_array`, string arrays through `_crush_string_array`, etc. +//! +//! # Python parity note: bool vs int +//! +//! Python's `True`/`False` are an int subclass, so a list `[True, False, 1]` +//! has `types == {bool, int}` but `[True, False]` has `types == {bool}`. +//! The Python code uses two checks to disambiguate: +//! 1. `has_bool` flag set during the type-walk +//! 2. `all(isinstance(i, bool) for i in items)` for pure-bool arrays +//! +//! The Rust `serde_json::Value` enum has separate `Bool` and `Number` +//! variants — no inheritance — so the disambiguation is naturally cleaner +//! here. We still walk every element (not a sample) to guarantee correct +//! classification on adversarial inputs. + +use serde_json::Value; + +/// JSON array element type classification. +/// +/// Mirrors Python's `ArrayType` enum at `smart_crusher.py:329-338`. The +/// string variants in `Display`/`Debug` match Python's lowercase `value=` +/// strings exactly, which is required for parity with serialized strategy +/// debug output (e.g. `"dict_array(100->10)"`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ArrayType { + /// `[{...}, {...}, ...]` — dict array, full statistical path. + DictArray, + /// `["a", "b", "c", ...]` — string array. + StringArray, + /// `[1, 2.5, 3, ...]` — number array (excludes bools). + NumberArray, + /// `[true, false, ...]` — pure bool array. + BoolArray, + /// `[[...], [...], ...]` — array of arrays. + NestedArray, + /// Anything else: heterogeneous or unclassifiable. + MixedArray, + /// `[]` — empty array. + Empty, +} + +impl ArrayType { + /// Lowercase string representation matching Python's `Enum.value`. + /// Used in strategy debug strings; must match Python exactly. + pub fn as_str(self) -> &'static str { + match self { + ArrayType::DictArray => "dict_array", + ArrayType::StringArray => "string_array", + ArrayType::NumberArray => "number_array", + ArrayType::BoolArray => "bool_array", + ArrayType::NestedArray => "nested_array", + ArrayType::MixedArray => "mixed_array", + ArrayType::Empty => "empty", + } + } +} + +/// Classify a JSON array by its element types. +/// +/// Walks every element (not a sample) to guarantee correct classification +/// even on adversarial inputs where the first few items hide a type +/// transition deeper in the list. `Value::is_*` is O(1), so the full +/// walk is fine. +/// +/// Returns `ArrayType::Empty` for an empty slice. +pub fn classify_array(items: &[Value]) -> ArrayType { + if items.is_empty() { + return ArrayType::Empty; + } + + // Track which Value variants we've seen. We collapse Number into + // either "int-like" or "float-like" once below; here we only need to + // know whether there's at least one of each high-level kind. + let mut has_bool = false; + let mut has_number = false; + let mut has_string = false; + let mut has_object = false; + let mut has_array = false; + let mut has_null = false; + + for item in items { + match item { + Value::Bool(_) => has_bool = true, + Value::Number(_) => has_number = true, + Value::String(_) => has_string = true, + Value::Object(_) => has_object = true, + Value::Array(_) => has_array = true, + Value::Null => has_null = true, + } + } + + // Pure bool array — Python's check is `all(isinstance(i, bool))`. + // Note Python `[True, False, 1]` evaluates to `types == {bool, int}` + // because bool is an int subclass; that maps to MixedArray here. + if has_bool && !has_number && !has_string && !has_object && !has_array && !has_null { + return ArrayType::BoolArray; + } + + // Pure dict array. + if has_object && !has_bool && !has_number && !has_string && !has_array && !has_null { + return ArrayType::DictArray; + } + + // Pure string array. + if has_string && !has_bool && !has_number && !has_object && !has_array && !has_null { + return ArrayType::StringArray; + } + + // Pure number array — Python explicitly excludes bool here. + if has_number && !has_bool && !has_string && !has_object && !has_array && !has_null { + return ArrayType::NumberArray; + } + + // Pure nested array. + if has_array && !has_bool && !has_number && !has_string && !has_object && !has_null { + return ArrayType::NestedArray; + } + + // Anything else — heterogeneous types, or types involving null. + ArrayType::MixedArray +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn empty_array() { + let items: Vec = vec![]; + assert_eq!(classify_array(&items), ArrayType::Empty); + } + + #[test] + fn pure_dict_array() { + let items = vec![json!({"a": 1}), json!({"b": 2})]; + assert_eq!(classify_array(&items), ArrayType::DictArray); + } + + #[test] + fn pure_string_array() { + let items = vec![json!("a"), json!("b"), json!("c")]; + assert_eq!(classify_array(&items), ArrayType::StringArray); + } + + #[test] + fn pure_number_array_int_and_float() { + let items = vec![json!(1), json!(2.5), json!(3)]; + assert_eq!(classify_array(&items), ArrayType::NumberArray); + } + + #[test] + fn pure_bool_array() { + let items = vec![json!(true), json!(false), json!(true)]; + assert_eq!(classify_array(&items), ArrayType::BoolArray); + } + + #[test] + fn nested_array() { + let items = vec![json!([1, 2]), json!([3, 4])]; + assert_eq!(classify_array(&items), ArrayType::NestedArray); + } + + #[test] + fn mixed_dict_and_string_is_mixed() { + let items = vec![json!({"a": 1}), json!("str")]; + assert_eq!(classify_array(&items), ArrayType::MixedArray); + } + + #[test] + fn bool_with_number_is_mixed_not_bool_or_number() { + // Python's `[True, False, 1]` matches `types <= {bool, int}` BUT + // fails `all(isinstance(i, bool))`, so falls through to NUMBER_ARRAY + // check which has `not has_bool` — fails. Then nested check fails. + // Final: MIXED_ARRAY. Same here. + let items = vec![json!(true), json!(false), json!(1)]; + assert_eq!(classify_array(&items), ArrayType::MixedArray); + } + + #[test] + fn null_in_array_is_mixed() { + // Python's `types == {dict}` check fails when None (NoneType) is + // present, so a dict array with one null falls to MIXED_ARRAY. + let items = vec![json!({"a": 1}), json!(null)]; + assert_eq!(classify_array(&items), ArrayType::MixedArray); + } + + #[test] + fn as_str_matches_python_values() { + // Strategy debug strings depend on these exact lowercase forms. + assert_eq!(ArrayType::DictArray.as_str(), "dict_array"); + assert_eq!(ArrayType::StringArray.as_str(), "string_array"); + assert_eq!(ArrayType::NumberArray.as_str(), "number_array"); + assert_eq!(ArrayType::BoolArray.as_str(), "bool_array"); + assert_eq!(ArrayType::NestedArray.as_str(), "nested_array"); + assert_eq!(ArrayType::MixedArray.as_str(), "mixed_array"); + assert_eq!(ArrayType::Empty.as_str(), "empty"); + } +} diff --git a/crates/headroom-core/src/transforms/smart_crusher/config.rs b/crates/headroom-core/src/transforms/smart_crusher/config.rs new file mode 100644 index 000000000..cb7959856 --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/config.rs @@ -0,0 +1,99 @@ +//! SmartCrusher configuration. +//! +//! Direct port of `SmartCrusherConfig` at `smart_crusher.py:927-957`. The +//! defaults must match Python exactly — they're consulted everywhere +//! during compression and any drift breaks parity fixtures. + +/// Configuration for SmartCrusher. +/// +/// SCHEMA-PRESERVING: Output contains only items from the original array. +/// No wrappers, no generated text, no metadata keys. (Python comment at +/// line 930-931.) +#[derive(Debug, Clone)] +pub struct SmartCrusherConfig { + pub enabled: bool, + /// Don't analyze arrays smaller than this. Default 5. + pub min_items_to_analyze: usize, + /// Only crush content with more than this many tokens. Default 200. + pub min_tokens_to_crush: usize, + /// Standard deviations from the mean to count as a change point. + /// Default 2.0. + pub variance_threshold: f64, + /// Below this unique-ratio, a field is treated as nearly constant. + /// Default 0.1. + pub uniqueness_threshold: f64, + /// Similarity score above which strings cluster together. Default 0.8. + pub similarity_threshold: f64, + /// Target maximum items in the output. Default 15. + pub max_items_after_crush: usize, + /// Whether to preserve detected change points. Default true. + pub preserve_change_points: bool, + /// Factor out fields with constant values across all items. Default + /// false (disabled — preserves original schema). + pub factor_out_constants: bool, + /// Include generated text summaries in output. Default false (disabled + /// — no generated text). + pub include_summaries: bool, + /// Use feedback hints to adjust compression aggressiveness. Default true. + pub use_feedback_hints: bool, + /// Minimum confidence required to apply TOIN recommendations. + /// Default 0.5. (Python LOW FIX #21.) + pub toin_confidence_threshold: f64, + /// Drop content-identical items before sampling. Default true. + pub dedup_identical_items: bool, + /// Fraction of K to allocate to the start of the array. Default 0.3. + pub first_fraction: f64, + /// Fraction of K to allocate to the end of the array. Default 0.15. + pub last_fraction: f64, +} + +impl Default for SmartCrusherConfig { + fn default() -> Self { + // These defaults must match smart_crusher.py:934-957 byte-for-byte. + SmartCrusherConfig { + enabled: true, + min_items_to_analyze: 5, + min_tokens_to_crush: 200, + variance_threshold: 2.0, + uniqueness_threshold: 0.1, + similarity_threshold: 0.8, + max_items_after_crush: 15, + preserve_change_points: true, + factor_out_constants: false, + include_summaries: false, + use_feedback_hints: true, + toin_confidence_threshold: 0.5, + dedup_identical_items: true, + first_fraction: 0.3, + last_fraction: 0.15, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_match_python() { + // Pin every default. Each field is consulted by some compression + // path and a drift would break parity. If Python ever changes a + // default, this test must be updated in lockstep. + let c = SmartCrusherConfig::default(); + assert!(c.enabled); + assert_eq!(c.min_items_to_analyze, 5); + assert_eq!(c.min_tokens_to_crush, 200); + assert_eq!(c.variance_threshold, 2.0); + assert_eq!(c.uniqueness_threshold, 0.1); + assert_eq!(c.similarity_threshold, 0.8); + assert_eq!(c.max_items_after_crush, 15); + assert!(c.preserve_change_points); + assert!(!c.factor_out_constants); + assert!(!c.include_summaries); + assert!(c.use_feedback_hints); + assert_eq!(c.toin_confidence_threshold, 0.5); + assert!(c.dedup_identical_items); + assert_eq!(c.first_fraction, 0.3); + assert_eq!(c.last_fraction, 0.15); + } +} diff --git a/crates/headroom-core/src/transforms/smart_crusher/hashing.rs b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs new file mode 100644 index 000000000..4a4a404de --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs @@ -0,0 +1,58 @@ +//! Field-name hashing for cache keys. +//! +//! Direct port of `_hash_field_name` (Python `smart_crusher.py:171-176`). +//! Used to generate stable cache keys for compression hints — must match +//! Python byte-for-byte or cache lookups will miss. + +use sha2::{Digest, Sha256}; + +/// SHA-256 of the UTF-8 bytes, hex-encoded, truncated to 16 chars. +/// +/// Python equivalent: `hashlib.sha256(field_name.encode()).hexdigest()[:16]`. +/// We use lowercase hex (the default for both Python and Rust's `sha2` +/// crate) — the test below pins this. +pub fn hash_field_name(field_name: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(field_name.as_bytes()); + let digest = hasher.finalize(); + // Convert to lowercase hex, then truncate to first 16 chars (8 bytes). + let hex = format!("{:x}", digest); + hex[..16].to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_python_sha256_truncated_to_16() { + // Verified against Python: hashlib.sha256(b"customer_id").hexdigest()[:16] + assert_eq!(hash_field_name("customer_id"), "1e38d67dbe8f47d2"); + } + + #[test] + fn empty_string() { + // Verified against Python: hashlib.sha256(b"").hexdigest()[:16] + assert_eq!(hash_field_name(""), "e3b0c44298fc1c14"); + } + + #[test] + fn unicode_field_name() { + // Verified against Python: hashlib.sha256("café".encode()).hexdigest()[:16] + // UTF-8 bytes for "café" are 63 61 66 c3 a9 — must encode same way. + assert_eq!(hash_field_name("café"), "850f7dc43910ff89"); + } + + #[test] + fn deterministic() { + // Same input → same output across calls. + assert_eq!(hash_field_name("test"), hash_field_name("test")); + } + + #[test] + fn output_length_is_16() { + // Always exactly 16 hex chars regardless of input length. + assert_eq!(hash_field_name("a").len(), 16); + assert_eq!(hash_field_name(&"x".repeat(1000)).len(), 16); + } +} diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rs new file mode 100644 index 000000000..9c408cb39 --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs @@ -0,0 +1,50 @@ +//! Smart statistical tool output compression — Rust port of +//! `headroom/transforms/smart_crusher.py`. +//! +//! # Stage 3c.1: like-for-like parity port +//! +//! This module is a literal Rust port of the Python `SmartCrusher` +//! implementation. The goal of Stage 3c.1 is **byte-equal output parity** for +//! every fixture in `tests/parity/fixtures/smart_crusher/`. Architectural +//! improvements (lossless-first, unified saliency score, structured CCR +//! markers, token budget) are deferred to Stage 3c.2 and tracked in +//! `~/Desktop/SmartCrusher-Architecture-Improvements.md`. +//! +//! # Bugs fixed in BOTH Python and Rust during 3c.1 +//! +//! Four defects in the Python source (`headroom/transforms/smart_crusher.py`) +//! were caught during port review. They're fixed in both languages +//! simultaneously so the parity fixtures continue to byte-match: +//! +//! - **k-split overshoot** (line 2722): `_compute_k_split` keeps 2 items when +//! `k_total = 1` because `max(1, round(k_total * fraction))` floors both +//! first and last to 1. Violates `max_items_after_crush`. +//! - **sequential-pattern false positive** (line 444): `_detect_sequential_pattern` +//! does `int("001")` and silently loses zero padding. Padded string IDs +//! misclassified as sequential numeric IDs. +//! - **rare-status detection short-circuit** (line 674): `_detect_rare_status_values` +//! exits early at >10 distinct values. Datasets with 50+ error codes lose +//! rare-error preservation. +//! - **percentile off-by-one** (line 2844): For `len < 8`, integer-division +//! percentile indices are off by one. Cosmetic — only affects strategy +//! debug strings. +//! +//! Each fix has a fixture entry in the parity harness and a corresponding +//! test in `tests/test_transforms/test_smart_crusher_bugs.py`. + +mod anchors; +mod classifier; +mod config; +mod hashing; +mod statistics; +mod types; + +pub use anchors::{extract_query_anchors, item_matches_anchors}; +pub use classifier::{classify_array, ArrayType}; +pub use config::SmartCrusherConfig; +pub use hashing::hash_field_name; +pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format}; +pub use types::{ + ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis, + FieldStats, +}; diff --git a/crates/headroom-core/src/transforms/smart_crusher/statistics.rs b/crates/headroom-core/src/transforms/smart_crusher/statistics.rs new file mode 100644 index 000000000..d3530ddd2 --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/statistics.rs @@ -0,0 +1,375 @@ +//! Statistical helpers for field characterization. +//! +//! Direct port of the helpers at `smart_crusher.py:378-481`. These are +//! used by the analyzer to classify fields (ID-like, score-like, etc.). +//! Detection logic is heuristic; small numeric drift between Python and +//! Rust would change classifications and break fixtures, so the math +//! here mirrors Python step-by-step. + +use serde_json::Value; +use std::collections::HashMap; + +/// Check if a string looks like a UUID. +/// +/// Direct port of `_is_uuid_format` (Python `smart_crusher.py:378-392`). +/// Format check only — no version-bit validation. Hex chars are lower +/// or upper case, matching Python. +pub fn is_uuid_format(value: &str) -> bool { + if value.len() != 36 { + return false; + } + + // Expected segment lengths: 8-4-4-4-12. + let parts: Vec<&str> = value.split('-').collect(); + if parts.len() != 5 { + return false; + } + let expected_lens = [8, 4, 4, 4, 12]; + for (part, &expected_len) in parts.iter().zip(expected_lens.iter()) { + if part.len() != expected_len { + return false; + } + for c in part.chars() { + if !c.is_ascii_hexdigit() { + return false; + } + } + } + true +} + +/// Shannon entropy of a string, normalized to `[0, 1]`. +/// +/// Direct port of `_calculate_string_entropy` (`smart_crusher.py:395-423`). +/// High entropy (>0.7) suggests random/ID-like content. Low entropy +/// (<0.3) suggests repetitive/predictable content. Used by ID detection. +/// +/// # Edge cases (matched to Python) +/// - Empty or single-character strings return `0.0`. +/// - All-identical chars: `freq` has 1 entry, `max_entropy = log2(min(1, n)) = 0.0`, +/// we return `0.0` to avoid division by zero. +pub fn calculate_string_entropy(s: &str) -> f64 { + // Python uses `len(s) < 2` and computes by character. Rust strings + // are UTF-8 so we iterate `chars()` to match Python's character-level + // semantics (Python iterates code points; Rust's `chars()` yields + // Unicode scalar values — same thing for non-surrogate text). + let n = s.chars().count(); + if n < 2 { + return 0.0; + } + + let mut freq: HashMap = HashMap::new(); + for c in s.chars() { + *freq.entry(c).or_insert(0) += 1; + } + + let length = n as f64; + let mut entropy = 0.0_f64; + for &count in freq.values() { + let p = count as f64 / length; + if p > 0.0 { + entropy -= p * p.log2(); + } + } + + // Normalize by the maximum possible entropy at this length: + // Python: max_entropy = log2(min(len(freq), length)) + let max_entropy = (freq.len().min(n) as f64).log2(); + if max_entropy > 0.0 { + entropy / max_entropy + } else { + 0.0 + } +} + +/// Detect if numeric values form a sequential pattern (like IDs: +/// 1, 2, 3, ...). +/// +/// Direct port of `_detect_sequential_pattern` (`smart_crusher.py:426-481`) +/// **with BUG #2 FIXED**. +/// +/// # Bug #2 — string-padding misclassification +/// Python's original implementation calls `int("001") == 1` and silently +/// loses zero padding, so a list of padded string IDs like +/// `["001", "002", ..., "100"]` looks like a sequential numeric pattern +/// when in reality it's a categorical string field where the padding +/// matters. The fix: when a value is a string that parses as a number, +/// flag the input as "had string-encoded numerics". If ALL parsed values +/// originated as strings, refuse to classify as a sequential numeric +/// pattern. Mixed numeric+string inputs still parse as sequential because +/// the unambiguous numeric values dominate the signal. +/// +/// This fix is applied in BOTH languages simultaneously (Python `smart_crusher.py` +/// gets the same fix in the same PR) so the parity fixtures continue to +/// match. Tests covering this bug live at +/// `tests/test_transforms/test_smart_crusher_bugs.py`. +/// +/// # Args +/// - `values`: items to inspect. +/// - `check_order`: when true, also require ascending order in the +/// original array (the Python flag — IDs are usually ascending in +/// source order, scores are usually descending). +pub fn detect_sequential_pattern(values: &[Value], check_order: bool) -> bool { + if values.len() < 5 { + return false; + } + + // Collect numeric values, tracking whether each value originated as + // a string. This is the BUG #2 fix: we still parse strings into + // numbers (so legitimate mixed-type fields work), but we'll refuse + // to flag the field as sequential if EVERY parseable value was a + // string. + let mut nums: Vec = Vec::new(); + let mut had_non_string_numeric = false; + + for v in values { + match v { + Value::Number(n) => { + if let Some(f) = n.as_f64() { + nums.push(f); + had_non_string_numeric = true; + } + } + Value::Bool(_) => { + // Python: `isinstance(v, int | float) and not isinstance(v, bool)` — + // bools are explicitly excluded. + } + Value::String(s) => { + // Python: `try: nums.append(int(v))`. `int("3.14")` raises + // in Python, so we mirror by trying integer parse first and + // only succeeding for pure-integer strings. + if let Ok(parsed) = s.parse::() { + nums.push(parsed as f64); + // BUG #2 fix: do NOT set had_non_string_numeric. + // If we later find this is the ONLY source of numeric + // values, we refuse to call it sequential. + } + } + _ => {} + } + } + + if nums.len() < 5 { + return false; + } + + // BUG #2 fix gate: if every numeric value originated as a string, + // the field is categorical (e.g. zero-padded codes); not sequential. + if !had_non_string_numeric { + return false; + } + + // Need at least 2 elements for pairwise comparison. (Python checks + // this redundantly after `len(nums) < 5`.) + if nums.len() < 2 { + return false; + } + + // Sort and compute pairwise diffs. + let mut sorted_nums = nums.clone(); + sorted_nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let diffs: Vec = sorted_nums + .windows(2) + .map(|w| w[1] - w[0]) + .collect(); + if diffs.is_empty() { + return false; + } + + let avg_diff: f64 = diffs.iter().sum::() / diffs.len() as f64; + if !(0.5..=2.0).contains(&avg_diff) { + return false; + } + + // Most diffs in [0.5, 2.0] => sequential candidate. + let consistent_count = diffs.iter().filter(|&&d| (0.5..=2.0).contains(&d)).count(); + let is_sequential = consistent_count as f64 / diffs.len() as f64 > 0.8; + if !is_sequential { + return false; + } + + if check_order { + // Python: ascending count over original (not-sorted) sequence. + // IDs ascend in array order; scores typically descend. + let ascending_count = nums + .windows(2) + .filter(|w| w[0] <= w[1]) + .count(); + let n_pairs = nums.len() - 1; + let is_ascending = ascending_count as f64 / n_pairs as f64 > 0.7; + return is_ascending; + } + + is_sequential +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + // ---------- is_uuid_format ---------- + + #[test] + fn uuid_format_canonical_lowercase() { + assert!(is_uuid_format("550e8400-e29b-41d4-a716-446655440000")); + } + + #[test] + fn uuid_format_uppercase() { + assert!(is_uuid_format("550E8400-E29B-41D4-A716-446655440000")); + } + + #[test] + fn uuid_format_wrong_length_rejected() { + assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000")); // 1 short + assert!(!is_uuid_format("550e8400-e29b-41d4-a716-4466554400000")); // 1 long + } + + #[test] + fn uuid_format_wrong_segment_count() { + assert!(!is_uuid_format("550e8400e29b41d4a716446655440000")); + } + + #[test] + fn uuid_format_non_hex_rejected() { + assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000z")); + } + + #[test] + fn uuid_format_empty_rejected() { + assert!(!is_uuid_format("")); + } + + // ---------- calculate_string_entropy ---------- + + #[test] + fn entropy_empty_string_is_zero() { + assert_eq!(calculate_string_entropy(""), 0.0); + } + + #[test] + fn entropy_single_char_is_zero() { + assert_eq!(calculate_string_entropy("a"), 0.0); + } + + #[test] + fn entropy_all_same_chars_is_zero() { + // "aaaa" — freq has 1 entry, max_entropy = log2(1) = 0.0, + // we return 0.0 from the guard. + assert_eq!(calculate_string_entropy("aaaa"), 0.0); + } + + #[test] + fn entropy_perfectly_uniform_normalized_to_one() { + // Two distinct chars, 50/50: raw entropy = 1.0, max = log2(2) = 1.0, + // normalized = 1.0. + let e = calculate_string_entropy("ab"); + assert!((e - 1.0).abs() < 1e-9); + } + + #[test] + fn entropy_mostly_repeated_low() { + // "aaaaaab" — 6/7 'a', 1/7 'b' — should be small. + let e = calculate_string_entropy("aaaaaab"); + assert!(e < 0.7); + } + + #[test] + fn entropy_high_for_random_looking_string() { + // Approximation of a UUID-ish hex string. Should be > 0.7. + let e = calculate_string_entropy("a3f7b2c9d8e1f4a7"); + assert!(e > 0.7); + } + + // ---------- detect_sequential_pattern ---------- + + #[test] + fn sequential_simple_int_ascending() { + let v: Vec = (1..=10).map(|i| json!(i)).collect(); + assert!(detect_sequential_pattern(&v, true)); + } + + #[test] + fn sequential_too_few_values() { + let v = vec![json!(1), json!(2), json!(3)]; + assert!(!detect_sequential_pattern(&v, true)); + } + + #[test] + fn sequential_random_numbers_not_detected() { + let v: Vec = vec![ + json!(100), + json!(2), + json!(85), + json!(7), + json!(43), + json!(17), + ]; + assert!(!detect_sequential_pattern(&v, true)); + } + + #[test] + fn sequential_descending_with_check_order_rejected() { + // Descending sequence — if check_order=true, must NOT be flagged + // as sequential (Python: scores descend, IDs ascend). + let v: Vec = (1..=10).rev().map(|i| json!(i)).collect(); + assert!(!detect_sequential_pattern(&v, true)); + } + + #[test] + fn sequential_descending_without_check_order_accepted() { + let v: Vec = (1..=10).rev().map(|i| json!(i)).collect(); + assert!(detect_sequential_pattern(&v, false)); + } + + #[test] + fn bug2_zero_padded_strings_no_longer_misclassified() { + // BUG #2: ["001", "002", ..., "010"] — Python's original code + // parsed each via int() and called this sequential. Fixed: every + // numeric value here originated as a string, so we refuse. + let v: Vec = (1..=10).map(|i| json!(format!("{:03}", i))).collect(); + assert!( + !detect_sequential_pattern(&v, true), + "BUG #2 fix: zero-padded string IDs must not be classified as sequential" + ); + } + + #[test] + fn bug2_mixed_string_and_int_still_detected() { + // Sanity check the fix doesn't break the legitimate case: a + // field that has BOTH genuine ints AND string-encoded ints + // should still be detected (the unambiguous ints dominate the + // signal). + let v = vec![ + json!(1), + json!(2), + json!("3"), + json!(4), + json!(5), + json!(6), + ]; + assert!(detect_sequential_pattern(&v, true)); + } + + #[test] + fn sequential_bools_excluded() { + // Python: `isinstance(v, int | float) and not isinstance(v, bool)` + // — bools never count as numeric. + let v = vec![ + json!(true), + json!(false), + json!(true), + json!(false), + json!(true), + json!(false), + ]; + assert!(!detect_sequential_pattern(&v, true)); + } + + #[test] + fn sequential_floats_with_unit_step() { + let v: Vec = (1..=10).map(|i| json!(i as f64)).collect(); + assert!(detect_sequential_pattern(&v, true)); + } +} diff --git a/crates/headroom-core/src/transforms/smart_crusher/types.rs b/crates/headroom-core/src/transforms/smart_crusher/types.rs new file mode 100644 index 000000000..5eb82a44f --- /dev/null +++ b/crates/headroom-core/src/transforms/smart_crusher/types.rs @@ -0,0 +1,244 @@ +//! Core data types for SmartCrusher. +//! +//! Direct port of the dataclasses in `smart_crusher.py:318-924`. These +//! mirror the Python shapes 1:1 so the PyO3 bridge in stage 3c.1b can +//! reconstruct Python dataclasses from the Rust output without a manual +//! field-by-field translator. + +use serde_json::Value; +use std::collections::BTreeMap; + +/// Compression strategies based on data patterns. +/// +/// Mirrors `CompressionStrategy` enum at `smart_crusher.py:318-326`. The +/// string variants must match Python's `Enum.value` exactly — they appear +/// in strategy debug strings (e.g. `"top_n(100->10)"`) and the parity +/// fixtures lock those bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CompressionStrategy { + /// No compression needed. + None, + /// Explicitly skip — not safe to crush. + Skip, + /// Time-series: keep change points, summarize stable runs. + TimeSeries, + /// Cluster-sample: dedupe similar items. + ClusterSample, + /// Top-N: keep highest-scored items. + TopN, + /// Smart-sample: statistical sampling with anchor-preservation. + SmartSample, +} + +impl CompressionStrategy { + /// Lowercase string matching Python's `Enum.value`. Pinned by the + /// parity fixtures — must not drift. + pub fn as_str(self) -> &'static str { + match self { + CompressionStrategy::None => "none", + CompressionStrategy::Skip => "skip", + CompressionStrategy::TimeSeries => "time_series", + CompressionStrategy::ClusterSample => "cluster", + CompressionStrategy::TopN => "top_n", + CompressionStrategy::SmartSample => "smart_sample", + } + } +} + +/// Statistics for a single field across array items. +/// +/// Mirrors the `FieldStats` dataclass at `smart_crusher.py:864-885`. +/// Field naming and Optional shape match Python exactly so the PyO3 +/// bridge can `from_dict`-reconstruct the Python dataclass. +#[derive(Debug, Clone)] +pub struct FieldStats { + pub name: String, + /// One of: `"numeric"`, `"string"`, `"boolean"`, `"object"`, `"array"`, + /// `"null"`. String literals match Python's `field_type` values. + pub field_type: String, + pub count: usize, + pub unique_count: usize, + pub unique_ratio: f64, + pub is_constant: bool, + pub constant_value: Option, + + // Numeric-specific + pub min_val: Option, + pub max_val: Option, + pub mean_val: Option, + pub variance: Option, + pub change_points: Vec, + + // String-specific + pub avg_length: Option, + /// Top values by frequency, descending. Bounded list so this stays + /// cheap to build and serialize. Same shape as Python's `list[tuple[str, int]]`. + pub top_values: Vec<(String, usize)>, +} + +/// Analysis of whether an array is safe to crush. +/// +/// Mirrors `CrushabilityAnalysis` at `smart_crusher.py:833-860`. The key +/// invariant: **if we don't have a reliable signal to determine which +/// items are important, we don't crush at all**. Signals include score +/// fields, error keywords, numeric anomalies, and low uniqueness. +#[derive(Debug, Clone)] +pub struct CrushabilityAnalysis { + pub crushable: bool, + pub confidence: f64, + pub reason: String, + pub signals_present: Vec, + pub signals_absent: Vec, + + // Detailed metrics (mirroring Python field-by-field) + pub has_id_field: bool, + pub id_uniqueness: f64, + pub avg_string_uniqueness: f64, + pub has_score_field: bool, + pub error_item_count: usize, + pub anomaly_count: usize, +} + +impl CrushabilityAnalysis { + /// Helper to build a "not crushable" verdict — used in several early + /// exits in `analyze_crushability`. Mirrors the Python pattern where + /// `crushable=False` paths don't bother filling in detail metrics. + pub fn skip(reason: impl Into, confidence: f64) -> Self { + CrushabilityAnalysis { + crushable: false, + confidence, + reason: reason.into(), + signals_present: Vec::new(), + signals_absent: Vec::new(), + has_id_field: false, + id_uniqueness: 0.0, + avg_string_uniqueness: 0.0, + has_score_field: false, + error_item_count: 0, + anomaly_count: 0, + } + } +} + +/// Complete analysis of an array. +/// +/// Mirrors `ArrayAnalysis` at `smart_crusher.py:887-897`. `field_stats` +/// uses `BTreeMap` for deterministic iteration order (Python's `dict` +/// preserves insertion order; `BTreeMap` gives us a stable sorted-by-key +/// order, which is fine for the parity fixtures because the analyzer +/// builds the map by iterating sorted keys). +#[derive(Debug, Clone)] +pub struct ArrayAnalysis { + pub item_count: usize, + pub field_stats: BTreeMap, + /// One of: `"time_series"`, `"logs"`, `"search_results"`, `"generic"`. + pub detected_pattern: String, + pub recommended_strategy: CompressionStrategy, + pub constant_fields: BTreeMap, + pub estimated_reduction: f64, + pub crushability: Option, +} + +/// Plan for how to compress an array. +/// +/// Mirrors `CompressionPlan` at `smart_crusher.py:900-910`. `keep_indices` +/// is the list of original-array indices that survive compression; +/// `summary_ranges` carries `(start, end, summary_dict)` for runs we +/// summarized rather than dropped (currently unused in the Python impl +/// but plumbed through for parity with the dataclass). +#[derive(Debug, Clone)] +pub struct CompressionPlan { + pub strategy: CompressionStrategy, + pub keep_indices: Vec, + pub constant_fields: BTreeMap, + /// `(start, end, summary)` triples for summarized runs. Python uses + /// `list[tuple[int, int, dict]]`; we use `Value` for the summary so + /// any JSON shape is representable. + pub summary_ranges: Vec<(usize, usize, Value)>, + pub cluster_field: Option, + pub sort_field: Option, + pub keep_count: usize, +} + +impl Default for CompressionPlan { + fn default() -> Self { + // Mirrors Python's @dataclass defaults at line 900-910. + CompressionPlan { + strategy: CompressionStrategy::None, + keep_indices: Vec::new(), + constant_fields: BTreeMap::new(), + summary_ranges: Vec::new(), + cluster_field: None, + sort_field: None, + keep_count: 10, + } + } +} + +/// Result from `SmartCrusher.crush()` — used by ContentRouter when +/// routing JSON arrays. Mirrors `CrushResult` at `smart_crusher.py:913-923`. +#[derive(Debug, Clone)] +pub struct CrushResult { + pub compressed: String, + pub original: String, + pub was_modified: bool, + pub strategy: String, +} + +impl CrushResult { + /// Pass-through result: same as input, no modification, strategy + /// `"passthrough"`. Used when content can't be compressed (not JSON, + /// too small, no crushable arrays, etc.). + pub fn passthrough(content: impl Into) -> Self { + let s = content.into(); + CrushResult { + compressed: s.clone(), + original: s, + was_modified: false, + strategy: "passthrough".to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compression_strategy_strings_match_python() { + // Strategy debug strings appear in the parity fixtures; these must + // not drift. If a value here changes, every fixture breaks. + assert_eq!(CompressionStrategy::None.as_str(), "none"); + assert_eq!(CompressionStrategy::Skip.as_str(), "skip"); + assert_eq!(CompressionStrategy::TimeSeries.as_str(), "time_series"); + assert_eq!(CompressionStrategy::ClusterSample.as_str(), "cluster"); + assert_eq!(CompressionStrategy::TopN.as_str(), "top_n"); + assert_eq!(CompressionStrategy::SmartSample.as_str(), "smart_sample"); + } + + #[test] + fn crushability_skip_helper() { + let r = CrushabilityAnalysis::skip("too small", 1.0); + assert!(!r.crushable); + assert_eq!(r.confidence, 1.0); + assert_eq!(r.reason, "too small"); + } + + #[test] + fn compression_plan_default_keep_count_matches_python() { + // Python's @dataclass default is `keep_count: int = 10`. + let p = CompressionPlan::default(); + assert_eq!(p.keep_count, 10); + assert_eq!(p.strategy, CompressionStrategy::None); + assert!(p.keep_indices.is_empty()); + } + + #[test] + fn crush_result_passthrough() { + let r = CrushResult::passthrough("hello"); + assert_eq!(r.compressed, "hello"); + assert_eq!(r.original, "hello"); + assert!(!r.was_modified); + assert_eq!(r.strategy, "passthrough"); + } +} diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json index 5413f88ea..c99f01e98 100644 --- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "headroom", - "version": "0.11.0", + "version": "0.10.17", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", "author": { "name": "Headroom Contributors", diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.json index 8b69de7cb..251e19767 100644 --- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json +++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "headroom", - "version": "0.11.0", + "version": "0.10.17", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", "author": { "name": "Headroom Contributors",