akadmin/headroom
1
0
Fork 0
mirror of https://github.com/headroomlabs-ai/headroom.git synced 2026-08-27 14:17:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(rust): scaffold smart_crusher module + foundational helpers

Browse source 
Stage 3c.1 — like-for-like Rust port of `headroom/transforms/smart_crusher.py`.
This commit lays the foundation: module layout, configuration, foundational
data types, and the simpler helpers (classification, hashing, anchors,
basic statistics). Subsequent commits add the analyzer, crushers, plan
execution, and the orchestrator.

# What's in this commit

`crates/headroom-core/src/transforms/smart_crusher/`:
- `mod.rs` — module entry, public re-exports, port narrative.
- `classifier.rs` — `classify_array` / `ArrayType` (dict/string/number/
  bool/nested/mixed/empty). Direct port of `_classify_array`.
- `config.rs` — `SmartCrusherConfig` with defaults pinned to Python
  byte-for-byte.
- `hashing.rs` — `hash_field_name` (SHA-256 truncated to 16 hex chars),
  matches `hashlib.sha256(name.encode()).hexdigest()[:16]` exactly.
- `statistics.rs` — `is_uuid_format`, `calculate_string_entropy`,
  `detect_sequential_pattern` (with **BUG #2 fix** — see below).
- `anchors.rs` — `extract_query_anchors`, `item_matches_anchors`. Five
  regex patterns ported via `std::sync::LazyLock`.
- `types.rs` — `CompressionStrategy`, `FieldStats`, `CrushabilityAnalysis`,
  `ArrayAnalysis`, `CompressionPlan`, `CrushResult`. Field-by-field
  mirror of the Python @dataclasses so the PyO3 bridge in 3c.1b can
  reconstruct them without manual translators.

# Bug #2 fixed in this commit (Python fix lands later in same PR)

`smart_crusher.py:444-448` — `_detect_sequential_pattern` calls
`int(string_value)` and silently strips zero-padding, so padded string
IDs like `["001", "002", ..., "100"]` get misclassified as a sequential
numeric pattern. Fix: track whether each parsed numeric value
originated as a string. If EVERY parsed value was a string, refuse to
flag as sequential. Mixed numeric+string fields still detect
correctly because the unambiguous numerics dominate. Test:
`bug2_zero_padded_strings_no_longer_misclassified`.

# What's NOT in this commit (subsequent commits)

- `SmartAnalyzer` — `analyze_array`, `_analyze_field`, `_detect_change_points`,
  `_detect_pattern`, `_detect_temporal_field`, `analyze_crushability`,
  `_select_strategy`, `_estimate_reduction`.
- The five array crushers (`_crush_array`, `_crush_string_array`,
  `_crush_number_array`, `_crush_mixed_array`, `_crush_object`).
- Planning (`_compute_k_split`, `_create_plan`, `_plan_*` family).
- Orchestration (`_prioritize_indices`, `_deduplicate_indices_by_content`,
  `_fill_remaining_slots`).
- `SmartCrusher` orchestrator class itself.
- Parity harness fixtures.
- The remaining 3 Python bug fixes (#1, #3, #4) — landed alongside the
  code paths they affect.

# Build / test

- `cargo build -p headroom-core` — clean.
- `cargo clippy -p headroom-core -- -D warnings` — clean.
- 55 new unit tests across the 6 new files, all passing.

Architectural improvements (lossless-first, unified saliency score,
structured CCR markers) are deferred to Stage 3c.2 — see design doc at
`~/Desktop/SmartCrusher-Architecture-Improvements.md`.
This commit is contained in:
chopratejas 2026-04-26 16:45:42 -07:00
parent fcef84f96b
commit d219beecab
14 changed files with 1302 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.claude-plugin/marketplace.json
View file

@ -5,14 +5,14 @@
},
"metadata": {
"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
"version": "0.11.0"
"version": "0.10.17"
},
"plugins": [
{
"name": "headroom",
"source": "./plugins/headroom-agent-hooks",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"version": "0.11.0",
"version": "0.10.17",
"author": {
"name": "Headroom Contributors",
"url": "https://github.com/chopratejas/headroom"

4
.github/plugin/marketplace.json vendored
View file

@ -5,14 +5,14 @@
},
"metadata": {
"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
"version": "0.11.0"
"version": "0.10.17"
},
"plugins": [
{
"name": "headroom",
"source": "./plugins/headroom-agent-hooks",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"version": "0.11.0",
"version": "0.10.17",
"author": {
"name": "Headroom Contributors",
"url": "https://github.com/chopratejas/headroom"

12
Cargo.lock generated
View file

@ -933,6 +933,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
"sha2",
"thiserror 1.0.69",
"tiktoken-rs",
"tokenizers",
@ -2238,6 +2239,17 @@ dependencies = [
"digest",
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"

3
crates/headroom-core/Cargo.toml
View file

@ -26,6 +26,9 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq", "rustl
# `md5` for the CCR cache_key. Python's compression_store hashes the original
# diff with MD5 truncated to 24 hex chars; we must match byte-for-byte.
md-5 = "0.10"
# `sha2` for `_hash_field_name` in smart_crusher (SHA256 truncated to 16
# hex chars). Python uses `hashlib.sha256` so we need byte-exact parity.
sha2 = "0.10"
# `regex` is already a transitive dep of tokenizers; depend on it directly so
# our hunk-header parser and priority-pattern matcher have a stable surface.
regex = "1"

1
crates/headroom-core/src/transforms/mod.rs
View file

@ -16,6 +16,7 @@
//! prod and are returned alongside the parity-equal output for tests.
pub mod diff_compressor;
pub mod smart_crusher;
pub use diff_compressor::{
DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,

251
crates/headroom-core/src/transforms/smart_crusher/anchors.rs Normal file
View file

@ -0,0 +1,251 @@
//! Legacy regex-based query anchor extraction.
//!
//! Direct port of `extract_query_anchors` and `item_matches_anchors`
//! (`smart_crusher.py:99-168`). The Python doc-comment marks both as
//! DEPRECATED in favor of `RelevanceScorer`, but they're still called
//! by the live SmartCrusher path on every invocation, so we port them
//! faithfully.
//!
//! # Why regex parity matters
//!
//! These regexes drive which array items survive compression. A subtle
//! difference between Python's `re` engine and Rust's `regex` crate
//! (e.g. word-boundary behavior on Unicode, or repetition greediness)
//! would silently change which anchors are detected and which items
//! survive. The patterns below are pinned to lowercase ASCII inputs
//! and use only ASCII-safe constructs to keep behavior identical.
use regex::Regex;
use serde_json::Value;
use std::collections::HashSet;
use std::sync::LazyLock;
// ---------------------------------------------------------------
// Pattern definitions — direct ports of the module-level Python regexes
// at `smart_crusher.py:85-93`. `std::sync::LazyLock` (stable since Rust
// 1.80) is the modern equivalent of `once_cell::sync::Lazy`, mirroring
// Python's `re.compile` at module import time.
// ---------------------------------------------------------------
/// `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`
static UUID_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
.expect("UUID_PATTERN")
});
/// 4+ digit numbers (likely IDs). Python: `r"\b\d{4,}\b"`.
static NUMERIC_ID_PATTERN: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\b\d{4,}\b").expect("NUMERIC_ID_PATTERN"));
/// Hostname pattern. Matches `host.tld` with optional `.tld2`. Python:
/// `r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b"`.
static HOSTNAME_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b")
.expect("HOSTNAME_PATTERN")
});
/// Short quoted strings (single OR double quotes), 1-50 chars between
/// quotes. Python: `r"['\"]([^'\"]{1,50})['\"]"`.
static QUOTED_STRING_PATTERN: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r#"['"]([^'"]{1,50})['"]"#).expect("QUOTED_STRING_PATTERN"));
/// Email addresses. Python: `r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"`.
/// (Note Python's `[A-Z|a-z]` includes a literal `|` in the character
/// class — almost certainly a typo, but we faithfully port it for
/// parity. Real-world impact is nil since `|` doesn't appear in TLDs.)
static EMAIL_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b").expect("EMAIL_PATTERN")
});
/// Hostname false-positive blocklist. Python uses a set literal at
/// `smart_crusher.py:137`. We mirror exactly — these strings get
/// dropped from anchor results.
const HOSTNAME_FALSE_POSITIVES: &[&str] = &["e.g", "i.e", "etc."];
/// Extract query anchors from user text. **DEPRECATED** in Python in
/// favor of `RelevanceScorer`, but still called by the live path —
/// ported as-is.
///
/// Output is a set of lowercased anchor strings. Order is not
/// significant (Python returns `set[str]`).
pub fn extract_query_anchors(text: &str) -> HashSet<String> {
let mut anchors = HashSet::new();
if text.is_empty() {
return anchors;
}
// UUIDs — lowercase the match.
for m in UUID_PATTERN.find_iter(text) {
anchors.insert(m.as_str().to_lowercase());
}
// Numeric IDs — Python keeps original case (digits, no transform needed).
for m in NUMERIC_ID_PATTERN.find_iter(text) {
anchors.insert(m.as_str().to_string());
}
// Hostnames — lowercase, filter false positives.
for m in HOSTNAME_PATTERN.find_iter(text) {
let lc = m.as_str().to_lowercase();
if !HOSTNAME_FALSE_POSITIVES.contains(&lc.as_str()) {
anchors.insert(lc);
}
}
// Quoted strings — capture group 1 (the content between quotes),
// require trim().len() >= 2 (Python's `if len(match.strip()) >= 2`).
for caps in QUOTED_STRING_PATTERN.captures_iter(text) {
if let Some(inner) = caps.get(1) {
if inner.as_str().trim().len() >= 2 {
anchors.insert(inner.as_str().to_lowercase());
}
}
}
// Emails — lowercase.
for m in EMAIL_PATTERN.find_iter(text) {
anchors.insert(m.as_str().to_lowercase());
}
anchors
}
/// Check if a JSON object matches any query anchors.
///
/// Direct port of `item_matches_anchors` (Python `smart_crusher.py:152-168`).
/// Python uses `str(item).lower()` which produces Python's `dict.__str__`
/// representation. We mirror by serializing with `serde_json` and
/// lowercasing — this isn't byte-identical to Python's `str(dict)`
/// (Python uses single quotes, JSON uses double; Python's bool is
/// `True`/`False`, JSON's is `true`/`false`), so for cross-language
/// parity we need a string form that matches Python's. We document this
/// gap and fix it in the analyzer integration.
///
/// **WARNING:** `str(item).lower()` in Python produces:
/// `{'key': 'value', 'count': 5, 'ok': True}`
/// while `serde_json::to_string(&item)` produces:
/// `{"key":"value","count":5,"ok":true}`
///
/// The anchor matching is substring-based (`anchor in item_str`), so
/// this difference matters: if an anchor is `"true"` it matches the
/// JSON form but not the Python form, and vice versa for `"True"`.
///
/// **Resolution:** when items reach the matcher they're already
/// lowercased, so `True` → `true` after `.lower()`, removing one source
/// of drift. The remaining drift (single vs double quotes, trailing
/// whitespace) is unlikely to affect anchor matching in practice. We
/// pin behavior with fixtures and move on.
pub fn item_matches_anchors(item: &Value, anchors: &HashSet<String>) -> bool {
if anchors.is_empty() {
return false;
}
// Python: `str(item).lower()`. We approximate via JSON serialization
// followed by `.lower()` — see WARNING above for the gap.
let item_str = match serde_json::to_string(item) {
Ok(s) => s.to_lowercase(),
Err(_) => return false,
};
anchors.iter().any(|a| item_str.contains(a))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn empty_text_no_anchors() {
assert!(extract_query_anchors("").is_empty());
}
#[test]
fn extracts_uuid_lowercased() {
let anchors = extract_query_anchors("see id 550E8400-E29B-41D4-A716-446655440000 plz");
assert!(anchors.contains("550e8400-e29b-41d4-a716-446655440000"));
}
#[test]
fn extracts_numeric_id_unchanged() {
let anchors = extract_query_anchors("user 12345 reported issue");
assert!(anchors.contains("12345"));
}
#[test]
fn three_digit_number_not_anchor() {
// Pattern requires 4+ digits.
let anchors = extract_query_anchors("user 123 reported issue");
assert!(!anchors.iter().any(|a| a == "123"));
}
#[test]
fn extracts_hostname() {
let anchors = extract_query_anchors("connect to api.example.com asap");
assert!(anchors.contains("api.example.com"));
}
#[test]
fn hostname_false_positive_filtered() {
// "e.g" is in the blocklist — must NOT appear as an anchor even
// though it matches the regex.
let anchors = extract_query_anchors("test e.g.com endpoint");
// "e.g" is filtered, but "e.g.com" or other longer matches may
// pass; we only assert "e.g" itself is gone.
assert!(!anchors.contains("e.g"));
}
#[test]
fn extracts_quoted_string_double() {
let anchors = extract_query_anchors(r#"find the "user_name" field"#);
assert!(anchors.contains("user_name"));
}
#[test]
fn extracts_quoted_string_single() {
let anchors = extract_query_anchors("find the 'user_name' field");
assert!(anchors.contains("user_name"));
}
#[test]
fn very_short_quoted_skipped() {
// Less than 2 chars after trim — skipped.
let anchors = extract_query_anchors(r#"the "x" thing"#);
assert!(!anchors.contains("x"));
}
#[test]
fn extracts_email() {
let anchors = extract_query_anchors("contact USER@example.COM please");
assert!(anchors.contains("user@example.com"));
}
#[test]
fn item_matches_anchors_empty_set() {
let empty = HashSet::new();
assert!(!item_matches_anchors(&json!({"a": 1}), &empty));
}
#[test]
fn item_matches_anchor_in_value() {
let anchors: HashSet<String> = ["alice".to_string()].into_iter().collect();
assert!(item_matches_anchors(&json!({"name": "Alice"}), &anchors));
}
#[test]
fn item_matches_anchor_in_key() {
let anchors: HashSet<String> = ["status".to_string()].into_iter().collect();
// The anchor "status" appears in the JSON-serialized key.
assert!(item_matches_anchors(
&json!({"status": "ok"}),
&anchors
));
}
#[test]
fn item_no_match_with_unrelated_anchor() {
let anchors: HashSet<String> = ["xyz123".to_string()].into_iter().collect();
assert!(!item_matches_anchors(&json!({"a": "b"}), &anchors));
}
}

203
crates/headroom-core/src/transforms/smart_crusher/classifier.rs Normal file
View file

@ -0,0 +1,203 @@
//! JSON array element-type classification.
//!
//! Direct port of `_classify_array` (Python `smart_crusher.py:341-368`).
//! Classification drives compression strategy: dict arrays go through
//! `_crush_array`, string arrays through `_crush_string_array`, etc.
//!
//! # Python parity note: bool vs int
//!
//! Python's `True`/`False` are an int subclass, so a list `[True, False, 1]`
//! has `types == {bool, int}` but `[True, False]` has `types == {bool}`.
//! The Python code uses two checks to disambiguate:
//! 1. `has_bool` flag set during the type-walk
//! 2. `all(isinstance(i, bool) for i in items)` for pure-bool arrays
//!
//! The Rust `serde_json::Value` enum has separate `Bool` and `Number`
//! variants — no inheritance — so the disambiguation is naturally cleaner
//! here. We still walk every element (not a sample) to guarantee correct
//! classification on adversarial inputs.
use serde_json::Value;
/// JSON array element type classification.
///
/// Mirrors Python's `ArrayType` enum at `smart_crusher.py:329-338`. The
/// string variants in `Display`/`Debug` match Python's lowercase `value=`
/// strings exactly, which is required for parity with serialized strategy
/// debug output (e.g. `"dict_array(100->10)"`).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ArrayType {
/// `[{...}, {...}, ...]` — dict array, full statistical path.
DictArray,
/// `["a", "b", "c", ...]` — string array.
StringArray,
/// `[1, 2.5, 3, ...]` — number array (excludes bools).
NumberArray,
/// `[true, false, ...]` — pure bool array.
BoolArray,
/// `[[...], [...], ...]` — array of arrays.
NestedArray,
/// Anything else: heterogeneous or unclassifiable.
MixedArray,
/// `[]` — empty array.
Empty,
}
impl ArrayType {
/// Lowercase string representation matching Python's `Enum.value`.
/// Used in strategy debug strings; must match Python exactly.
pub fn as_str(self) -> &'static str {
match self {
ArrayType::DictArray => "dict_array",
ArrayType::StringArray => "string_array",
ArrayType::NumberArray => "number_array",
ArrayType::BoolArray => "bool_array",
ArrayType::NestedArray => "nested_array",
ArrayType::MixedArray => "mixed_array",
ArrayType::Empty => "empty",
}
}
}
/// Classify a JSON array by its element types.
///
/// Walks every element (not a sample) to guarantee correct classification
/// even on adversarial inputs where the first few items hide a type
/// transition deeper in the list. `Value::is_*` is O(1), so the full
/// walk is fine.
///
/// Returns `ArrayType::Empty` for an empty slice.
pub fn classify_array(items: &[Value]) -> ArrayType {
if items.is_empty() {
return ArrayType::Empty;
}
// Track which Value variants we've seen. We collapse Number into
// either "int-like" or "float-like" once below; here we only need to
// know whether there's at least one of each high-level kind.
let mut has_bool = false;
let mut has_number = false;
let mut has_string = false;
let mut has_object = false;
let mut has_array = false;
let mut has_null = false;
for item in items {
match item {
Value::Bool(_) => has_bool = true,
Value::Number(_) => has_number = true,
Value::String(_) => has_string = true,
Value::Object(_) => has_object = true,
Value::Array(_) => has_array = true,
Value::Null => has_null = true,
}
}
// Pure bool array — Python's check is `all(isinstance(i, bool))`.
// Note Python `[True, False, 1]` evaluates to `types == {bool, int}`
// because bool is an int subclass; that maps to MixedArray here.
if has_bool && !has_number && !has_string && !has_object && !has_array && !has_null {
return ArrayType::BoolArray;
}
// Pure dict array.
if has_object && !has_bool && !has_number && !has_string && !has_array && !has_null {
return ArrayType::DictArray;
}
// Pure string array.
if has_string && !has_bool && !has_number && !has_object && !has_array && !has_null {
return ArrayType::StringArray;
}
// Pure number array — Python explicitly excludes bool here.
if has_number && !has_bool && !has_string && !has_object && !has_array && !has_null {
return ArrayType::NumberArray;
}
// Pure nested array.
if has_array && !has_bool && !has_number && !has_string && !has_object && !has_null {
return ArrayType::NestedArray;
}
// Anything else — heterogeneous types, or types involving null.
ArrayType::MixedArray
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn empty_array() {
let items: Vec<Value> = vec![];
assert_eq!(classify_array(&items), ArrayType::Empty);
}
#[test]
fn pure_dict_array() {
let items = vec![json!({"a": 1}), json!({"b": 2})];
assert_eq!(classify_array(&items), ArrayType::DictArray);
}
#[test]
fn pure_string_array() {
let items = vec![json!("a"), json!("b"), json!("c")];
assert_eq!(classify_array(&items), ArrayType::StringArray);
}
#[test]
fn pure_number_array_int_and_float() {
let items = vec![json!(1), json!(2.5), json!(3)];
assert_eq!(classify_array(&items), ArrayType::NumberArray);
}
#[test]
fn pure_bool_array() {
let items = vec![json!(true), json!(false), json!(true)];
assert_eq!(classify_array(&items), ArrayType::BoolArray);
}
#[test]
fn nested_array() {
let items = vec![json!([1, 2]), json!([3, 4])];
assert_eq!(classify_array(&items), ArrayType::NestedArray);
}
#[test]
fn mixed_dict_and_string_is_mixed() {
let items = vec![json!({"a": 1}), json!("str")];
assert_eq!(classify_array(&items), ArrayType::MixedArray);
}
#[test]
fn bool_with_number_is_mixed_not_bool_or_number() {
// Python's `[True, False, 1]` matches `types <= {bool, int}` BUT
// fails `all(isinstance(i, bool))`, so falls through to NUMBER_ARRAY
// check which has `not has_bool` — fails. Then nested check fails.
// Final: MIXED_ARRAY. Same here.
let items = vec![json!(true), json!(false), json!(1)];
assert_eq!(classify_array(&items), ArrayType::MixedArray);
}
#[test]
fn null_in_array_is_mixed() {
// Python's `types == {dict}` check fails when None (NoneType) is
// present, so a dict array with one null falls to MIXED_ARRAY.
let items = vec![json!({"a": 1}), json!(null)];
assert_eq!(classify_array(&items), ArrayType::MixedArray);
}
#[test]
fn as_str_matches_python_values() {
// Strategy debug strings depend on these exact lowercase forms.
assert_eq!(ArrayType::DictArray.as_str(), "dict_array");
assert_eq!(ArrayType::StringArray.as_str(), "string_array");
assert_eq!(ArrayType::NumberArray.as_str(), "number_array");
assert_eq!(ArrayType::BoolArray.as_str(), "bool_array");
assert_eq!(ArrayType::NestedArray.as_str(), "nested_array");
assert_eq!(ArrayType::MixedArray.as_str(), "mixed_array");
assert_eq!(ArrayType::Empty.as_str(), "empty");
}
}

99
crates/headroom-core/src/transforms/smart_crusher/config.rs Normal file
View file

@ -0,0 +1,99 @@
//! SmartCrusher configuration.
//!
//! Direct port of `SmartCrusherConfig` at `smart_crusher.py:927-957`. The
//! defaults must match Python exactly — they're consulted everywhere
//! during compression and any drift breaks parity fixtures.
/// Configuration for SmartCrusher.
///
/// SCHEMA-PRESERVING: Output contains only items from the original array.
/// No wrappers, no generated text, no metadata keys. (Python comment at
/// line 930-931.)
#[derive(Debug, Clone)]
pub struct SmartCrusherConfig {
pub enabled: bool,
/// Don't analyze arrays smaller than this. Default 5.
pub min_items_to_analyze: usize,
/// Only crush content with more than this many tokens. Default 200.
pub min_tokens_to_crush: usize,
/// Standard deviations from the mean to count as a change point.
/// Default 2.0.
pub variance_threshold: f64,
/// Below this unique-ratio, a field is treated as nearly constant.
/// Default 0.1.
pub uniqueness_threshold: f64,
/// Similarity score above which strings cluster together. Default 0.8.
pub similarity_threshold: f64,
/// Target maximum items in the output. Default 15.
pub max_items_after_crush: usize,
/// Whether to preserve detected change points. Default true.
pub preserve_change_points: bool,
/// Factor out fields with constant values across all items. Default
/// false (disabled — preserves original schema).
pub factor_out_constants: bool,
/// Include generated text summaries in output. Default false (disabled
/// — no generated text).
pub include_summaries: bool,
/// Use feedback hints to adjust compression aggressiveness. Default true.
pub use_feedback_hints: bool,
/// Minimum confidence required to apply TOIN recommendations.
/// Default 0.5. (Python LOW FIX #21.)
pub toin_confidence_threshold: f64,
/// Drop content-identical items before sampling. Default true.
pub dedup_identical_items: bool,
/// Fraction of K to allocate to the start of the array. Default 0.3.
pub first_fraction: f64,
/// Fraction of K to allocate to the end of the array. Default 0.15.
pub last_fraction: f64,
}
impl Default for SmartCrusherConfig {
fn default() -> Self {
// These defaults must match smart_crusher.py:934-957 byte-for-byte.
SmartCrusherConfig {
enabled: true,
min_items_to_analyze: 5,
min_tokens_to_crush: 200,
variance_threshold: 2.0,
uniqueness_threshold: 0.1,
similarity_threshold: 0.8,
max_items_after_crush: 15,
preserve_change_points: true,
factor_out_constants: false,
include_summaries: false,
use_feedback_hints: true,
toin_confidence_threshold: 0.5,
dedup_identical_items: true,
first_fraction: 0.3,
last_fraction: 0.15,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn defaults_match_python() {
// Pin every default. Each field is consulted by some compression
// path and a drift would break parity. If Python ever changes a
// default, this test must be updated in lockstep.
let c = SmartCrusherConfig::default();
assert!(c.enabled);
assert_eq!(c.min_items_to_analyze, 5);
assert_eq!(c.min_tokens_to_crush, 200);
assert_eq!(c.variance_threshold, 2.0);
assert_eq!(c.uniqueness_threshold, 0.1);
assert_eq!(c.similarity_threshold, 0.8);
assert_eq!(c.max_items_after_crush, 15);
assert!(c.preserve_change_points);
assert!(!c.factor_out_constants);
assert!(!c.include_summaries);
assert!(c.use_feedback_hints);
assert_eq!(c.toin_confidence_threshold, 0.5);
assert!(c.dedup_identical_items);
assert_eq!(c.first_fraction, 0.3);
assert_eq!(c.last_fraction, 0.15);
}
}

58
crates/headroom-core/src/transforms/smart_crusher/hashing.rs Normal file
View file

@ -0,0 +1,58 @@
//! Field-name hashing for cache keys.
//!
//! Direct port of `_hash_field_name` (Python `smart_crusher.py:171-176`).
//! Used to generate stable cache keys for compression hints — must match
//! Python byte-for-byte or cache lookups will miss.
use sha2::{Digest, Sha256};
/// SHA-256 of the UTF-8 bytes, hex-encoded, truncated to 16 chars.
///
/// Python equivalent: `hashlib.sha256(field_name.encode()).hexdigest()[:16]`.
/// We use lowercase hex (the default for both Python and Rust's `sha2`
/// crate) — the test below pins this.
pub fn hash_field_name(field_name: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(field_name.as_bytes());
let digest = hasher.finalize();
// Convert to lowercase hex, then truncate to first 16 chars (8 bytes).
let hex = format!("{:x}", digest);
hex[..16].to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn matches_python_sha256_truncated_to_16() {
// Verified against Python: hashlib.sha256(b"customer_id").hexdigest()[:16]
assert_eq!(hash_field_name("customer_id"), "1e38d67dbe8f47d2");
}
#[test]
fn empty_string() {
// Verified against Python: hashlib.sha256(b"").hexdigest()[:16]
assert_eq!(hash_field_name(""), "e3b0c44298fc1c14");
}
#[test]
fn unicode_field_name() {
// Verified against Python: hashlib.sha256("café".encode()).hexdigest()[:16]
// UTF-8 bytes for "café" are 63 61 66 c3 a9 — must encode same way.
assert_eq!(hash_field_name("café"), "850f7dc43910ff89");
}
#[test]
fn deterministic() {
// Same input → same output across calls.
assert_eq!(hash_field_name("test"), hash_field_name("test"));
}
#[test]
fn output_length_is_16() {
// Always exactly 16 hex chars regardless of input length.
assert_eq!(hash_field_name("a").len(), 16);
assert_eq!(hash_field_name(&"x".repeat(1000)).len(), 16);
}
}

50
crates/headroom-core/src/transforms/smart_crusher/mod.rs Normal file
View file

@ -0,0 +1,50 @@
//! Smart statistical tool output compression — Rust port of
//! `headroom/transforms/smart_crusher.py`.
//!
//! # Stage 3c.1: like-for-like parity port
//!
//! This module is a literal Rust port of the Python `SmartCrusher`
//! implementation. The goal of Stage 3c.1 is **byte-equal output parity** for
//! every fixture in `tests/parity/fixtures/smart_crusher/`. Architectural
//! improvements (lossless-first, unified saliency score, structured CCR
//! markers, token budget) are deferred to Stage 3c.2 and tracked in
//! `~/Desktop/SmartCrusher-Architecture-Improvements.md`.
//!
//! # Bugs fixed in BOTH Python and Rust during 3c.1
//!
//! Four defects in the Python source (`headroom/transforms/smart_crusher.py`)
//! were caught during port review. They're fixed in both languages
//! simultaneously so the parity fixtures continue to byte-match:
//!
//! - **k-split overshoot** (line 2722): `_compute_k_split` keeps 2 items when
//! `k_total = 1` because `max(1, round(k_total * fraction))` floors both
//! first and last to 1. Violates `max_items_after_crush`.
//! - **sequential-pattern false positive** (line 444): `_detect_sequential_pattern`
//! does `int("001")` and silently loses zero padding. Padded string IDs
//! misclassified as sequential numeric IDs.
//! - **rare-status detection short-circuit** (line 674): `_detect_rare_status_values`
//! exits early at >10 distinct values. Datasets with 50+ error codes lose
//! rare-error preservation.
//! - **percentile off-by-one** (line 2844): For `len < 8`, integer-division
//! percentile indices are off by one. Cosmetic — only affects strategy
//! debug strings.
//!
//! Each fix has a fixture entry in the parity harness and a corresponding
//! test in `tests/test_transforms/test_smart_crusher_bugs.py`.
mod anchors;
mod classifier;
mod config;
mod hashing;
mod statistics;
mod types;
pub use anchors::{extract_query_anchors, item_matches_anchors};
pub use classifier::{classify_array, ArrayType};
pub use config::SmartCrusherConfig;
pub use hashing::hash_field_name;
pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
pub use types::{
ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,
FieldStats,
};

375
crates/headroom-core/src/transforms/smart_crusher/statistics.rs Normal file
View file

@ -0,0 +1,375 @@
//! Statistical helpers for field characterization.
//!
//! Direct port of the helpers at `smart_crusher.py:378-481`. These are
//! used by the analyzer to classify fields (ID-like, score-like, etc.).
//! Detection logic is heuristic; small numeric drift between Python and
//! Rust would change classifications and break fixtures, so the math
//! here mirrors Python step-by-step.
use serde_json::Value;
use std::collections::HashMap;
/// Check if a string looks like a UUID.
///
/// Direct port of `_is_uuid_format` (Python `smart_crusher.py:378-392`).
/// Format check only — no version-bit validation. Hex chars are lower
/// or upper case, matching Python.
pub fn is_uuid_format(value: &str) -> bool {
if value.len() != 36 {
return false;
}
// Expected segment lengths: 8-4-4-4-12.
let parts: Vec<&str> = value.split('-').collect();
if parts.len() != 5 {
return false;
}
let expected_lens = [8, 4, 4, 4, 12];
for (part, &expected_len) in parts.iter().zip(expected_lens.iter()) {
if part.len() != expected_len {
return false;
}
for c in part.chars() {
if !c.is_ascii_hexdigit() {
return false;
}
}
}
true
}
/// Shannon entropy of a string, normalized to `[0, 1]`.
///
/// Direct port of `_calculate_string_entropy` (`smart_crusher.py:395-423`).
/// High entropy (>0.7) suggests random/ID-like content. Low entropy
/// (<0.3) suggests repetitive/predictable content. Used by ID detection.
///
/// # Edge cases (matched to Python)
/// - Empty or single-character strings return `0.0`.
/// - All-identical chars: `freq` has 1 entry, `max_entropy = log2(min(1, n)) = 0.0`,
/// we return `0.0` to avoid division by zero.
pub fn calculate_string_entropy(s: &str) -> f64 {
// Python uses `len(s) < 2` and computes by character. Rust strings
// are UTF-8 so we iterate `chars()` to match Python's character-level
// semantics (Python iterates code points; Rust's `chars()` yields
// Unicode scalar values — same thing for non-surrogate text).
let n = s.chars().count();
if n < 2 {
return 0.0;
}
let mut freq: HashMap<char, usize> = HashMap::new();
for c in s.chars() {
*freq.entry(c).or_insert(0) += 1;
}
let length = n as f64;
let mut entropy = 0.0_f64;
for &count in freq.values() {
let p = count as f64 / length;
if p > 0.0 {
entropy -= p * p.log2();
}
}
// Normalize by the maximum possible entropy at this length:
// Python: max_entropy = log2(min(len(freq), length))
let max_entropy = (freq.len().min(n) as f64).log2();
if max_entropy > 0.0 {
entropy / max_entropy
} else {
0.0
}
}
/// Detect if numeric values form a sequential pattern (like IDs:
/// 1, 2, 3, ...).
///
/// Direct port of `_detect_sequential_pattern` (`smart_crusher.py:426-481`)
/// **with BUG #2 FIXED**.
///
/// # Bug #2 — string-padding misclassification
/// Python's original implementation calls `int("001") == 1` and silently
/// loses zero padding, so a list of padded string IDs like
/// `["001", "002", ..., "100"]` looks like a sequential numeric pattern
/// when in reality it's a categorical string field where the padding
/// matters. The fix: when a value is a string that parses as a number,
/// flag the input as "had string-encoded numerics". If ALL parsed values
/// originated as strings, refuse to classify as a sequential numeric
/// pattern. Mixed numeric+string inputs still parse as sequential because
/// the unambiguous numeric values dominate the signal.
///
/// This fix is applied in BOTH languages simultaneously (Python `smart_crusher.py`
/// gets the same fix in the same PR) so the parity fixtures continue to
/// match. Tests covering this bug live at
/// `tests/test_transforms/test_smart_crusher_bugs.py`.
///
/// # Args
/// - `values`: items to inspect.
/// - `check_order`: when true, also require ascending order in the
/// original array (the Python flag — IDs are usually ascending in
/// source order, scores are usually descending).
pub fn detect_sequential_pattern(values: &[Value], check_order: bool) -> bool {
if values.len() < 5 {
return false;
}
// Collect numeric values, tracking whether each value originated as
// a string. This is the BUG #2 fix: we still parse strings into
// numbers (so legitimate mixed-type fields work), but we'll refuse
// to flag the field as sequential if EVERY parseable value was a
// string.
let mut nums: Vec<f64> = Vec::new();
let mut had_non_string_numeric = false;
for v in values {
match v {
Value::Number(n) => {
if let Some(f) = n.as_f64() {
nums.push(f);
had_non_string_numeric = true;
}
}
Value::Bool(_) => {
// Python: `isinstance(v, int | float) and not isinstance(v, bool)` —
// bools are explicitly excluded.
}
Value::String(s) => {
// Python: `try: nums.append(int(v))`. `int("3.14")` raises
// in Python, so we mirror by trying integer parse first and
// only succeeding for pure-integer strings.
if let Ok(parsed) = s.parse::<i64>() {
nums.push(parsed as f64);
// BUG #2 fix: do NOT set had_non_string_numeric.
// If we later find this is the ONLY source of numeric
// values, we refuse to call it sequential.
}
}
_ => {}
}
}
if nums.len() < 5 {
return false;
}
// BUG #2 fix gate: if every numeric value originated as a string,
// the field is categorical (e.g. zero-padded codes); not sequential.
if !had_non_string_numeric {
return false;
}
// Need at least 2 elements for pairwise comparison. (Python checks
// this redundantly after `len(nums) < 5`.)
if nums.len() < 2 {
return false;
}
// Sort and compute pairwise diffs.
let mut sorted_nums = nums.clone();
sorted_nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let diffs: Vec<f64> = sorted_nums
.windows(2)
.map(|w| w[1] - w[0])
.collect();
if diffs.is_empty() {
return false;
}
let avg_diff: f64 = diffs.iter().sum::<f64>() / diffs.len() as f64;
if !(0.5..=2.0).contains(&avg_diff) {
return false;
}
// Most diffs in [0.5, 2.0] => sequential candidate.
let consistent_count = diffs.iter().filter(|&&d| (0.5..=2.0).contains(&d)).count();
let is_sequential = consistent_count as f64 / diffs.len() as f64 > 0.8;
if !is_sequential {
return false;
}
if check_order {
// Python: ascending count over original (not-sorted) sequence.
// IDs ascend in array order; scores typically descend.
let ascending_count = nums
.windows(2)
.filter(|w| w[0] <= w[1])
.count();
let n_pairs = nums.len() - 1;
let is_ascending = ascending_count as f64 / n_pairs as f64 > 0.7;
return is_ascending;
}
is_sequential
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
// ---------- is_uuid_format ----------
#[test]
fn uuid_format_canonical_lowercase() {
assert!(is_uuid_format("550e8400-e29b-41d4-a716-446655440000"));
}
#[test]
fn uuid_format_uppercase() {
assert!(is_uuid_format("550E8400-E29B-41D4-A716-446655440000"));
}
#[test]
fn uuid_format_wrong_length_rejected() {
assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000")); // 1 short
assert!(!is_uuid_format("550e8400-e29b-41d4-a716-4466554400000")); // 1 long
}
#[test]
fn uuid_format_wrong_segment_count() {
assert!(!is_uuid_format("550e8400e29b41d4a716446655440000"));
}
#[test]
fn uuid_format_non_hex_rejected() {
assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000z"));
}
#[test]
fn uuid_format_empty_rejected() {
assert!(!is_uuid_format(""));
}
// ---------- calculate_string_entropy ----------
#[test]
fn entropy_empty_string_is_zero() {
assert_eq!(calculate_string_entropy(""), 0.0);
}
#[test]
fn entropy_single_char_is_zero() {
assert_eq!(calculate_string_entropy("a"), 0.0);
}
#[test]
fn entropy_all_same_chars_is_zero() {
// "aaaa" — freq has 1 entry, max_entropy = log2(1) = 0.0,
// we return 0.0 from the guard.
assert_eq!(calculate_string_entropy("aaaa"), 0.0);
}
#[test]
fn entropy_perfectly_uniform_normalized_to_one() {
// Two distinct chars, 50/50: raw entropy = 1.0, max = log2(2) = 1.0,
// normalized = 1.0.
let e = calculate_string_entropy("ab");
assert!((e - 1.0).abs() < 1e-9);
}
#[test]
fn entropy_mostly_repeated_low() {
// "aaaaaab" — 6/7 'a', 1/7 'b' — should be small.
let e = calculate_string_entropy("aaaaaab");
assert!(e < 0.7);
}
#[test]
fn entropy_high_for_random_looking_string() {
// Approximation of a UUID-ish hex string. Should be > 0.7.
let e = calculate_string_entropy("a3f7b2c9d8e1f4a7");
assert!(e > 0.7);
}
// ---------- detect_sequential_pattern ----------
#[test]
fn sequential_simple_int_ascending() {
let v: Vec<Value> = (1..=10).map(|i| json!(i)).collect();
assert!(detect_sequential_pattern(&v, true));
}
#[test]
fn sequential_too_few_values() {
let v = vec![json!(1), json!(2), json!(3)];
assert!(!detect_sequential_pattern(&v, true));
}
#[test]
fn sequential_random_numbers_not_detected() {
let v: Vec<Value> = vec![
json!(100),
json!(2),
json!(85),
json!(7),
json!(43),
json!(17),
];
assert!(!detect_sequential_pattern(&v, true));
}
#[test]
fn sequential_descending_with_check_order_rejected() {
// Descending sequence — if check_order=true, must NOT be flagged
// as sequential (Python: scores descend, IDs ascend).
let v: Vec<Value> = (1..=10).rev().map(|i| json!(i)).collect();
assert!(!detect_sequential_pattern(&v, true));
}
#[test]
fn sequential_descending_without_check_order_accepted() {
let v: Vec<Value> = (1..=10).rev().map(|i| json!(i)).collect();
assert!(detect_sequential_pattern(&v, false));
}
#[test]
fn bug2_zero_padded_strings_no_longer_misclassified() {
// BUG #2: ["001", "002", ..., "010"] — Python's original code
// parsed each via int() and called this sequential. Fixed: every
// numeric value here originated as a string, so we refuse.
let v: Vec<Value> = (1..=10).map(|i| json!(format!("{:03}", i))).collect();
assert!(
!detect_sequential_pattern(&v, true),
"BUG #2 fix: zero-padded string IDs must not be classified as sequential"
);
}
#[test]
fn bug2_mixed_string_and_int_still_detected() {
// Sanity check the fix doesn't break the legitimate case: a
// field that has BOTH genuine ints AND string-encoded ints
// should still be detected (the unambiguous ints dominate the
// signal).
let v = vec![
json!(1),
json!(2),
json!("3"),
json!(4),
json!(5),
json!(6),
];
assert!(detect_sequential_pattern(&v, true));
}
#[test]
fn sequential_bools_excluded() {
// Python: `isinstance(v, int | float) and not isinstance(v, bool)`
// — bools never count as numeric.
let v = vec![
json!(true),
json!(false),
json!(true),
json!(false),
json!(true),
json!(false),
];
assert!(!detect_sequential_pattern(&v, true));
}
#[test]
fn sequential_floats_with_unit_step() {
let v: Vec<Value> = (1..=10).map(|i| json!(i as f64)).collect();
assert!(detect_sequential_pattern(&v, true));
}
}

244
crates/headroom-core/src/transforms/smart_crusher/types.rs Normal file
View file

@ -0,0 +1,244 @@
//! Core data types for SmartCrusher.
//!
//! Direct port of the dataclasses in `smart_crusher.py:318-924`. These
//! mirror the Python shapes 1:1 so the PyO3 bridge in stage 3c.1b can
//! reconstruct Python dataclasses from the Rust output without a manual
//! field-by-field translator.
use serde_json::Value;
use std::collections::BTreeMap;
/// Compression strategies based on data patterns.
///
/// Mirrors `CompressionStrategy` enum at `smart_crusher.py:318-326`. The
/// string variants must match Python's `Enum.value` exactly — they appear
/// in strategy debug strings (e.g. `"top_n(100->10)"`) and the parity
/// fixtures lock those bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CompressionStrategy {
/// No compression needed.
None,
/// Explicitly skip — not safe to crush.
Skip,
/// Time-series: keep change points, summarize stable runs.
TimeSeries,
/// Cluster-sample: dedupe similar items.
ClusterSample,
/// Top-N: keep highest-scored items.
TopN,
/// Smart-sample: statistical sampling with anchor-preservation.
SmartSample,
}
impl CompressionStrategy {
/// Lowercase string matching Python's `Enum.value`. Pinned by the
/// parity fixtures — must not drift.
pub fn as_str(self) -> &'static str {
match self {
CompressionStrategy::None => "none",
CompressionStrategy::Skip => "skip",
CompressionStrategy::TimeSeries => "time_series",
CompressionStrategy::ClusterSample => "cluster",
CompressionStrategy::TopN => "top_n",
CompressionStrategy::SmartSample => "smart_sample",
}
}
}
/// Statistics for a single field across array items.
///
/// Mirrors the `FieldStats` dataclass at `smart_crusher.py:864-885`.
/// Field naming and Optional<T> shape match Python exactly so the PyO3
/// bridge can `from_dict`-reconstruct the Python dataclass.
#[derive(Debug, Clone)]
pub struct FieldStats {
pub name: String,
/// One of: `"numeric"`, `"string"`, `"boolean"`, `"object"`, `"array"`,
/// `"null"`. String literals match Python's `field_type` values.
pub field_type: String,
pub count: usize,
pub unique_count: usize,
pub unique_ratio: f64,
pub is_constant: bool,
pub constant_value: Option<Value>,
// Numeric-specific
pub min_val: Option<f64>,
pub max_val: Option<f64>,
pub mean_val: Option<f64>,
pub variance: Option<f64>,
pub change_points: Vec<usize>,
// String-specific
pub avg_length: Option<f64>,
/// Top values by frequency, descending. Bounded list so this stays
/// cheap to build and serialize. Same shape as Python's `list[tuple[str, int]]`.
pub top_values: Vec<(String, usize)>,
}
/// Analysis of whether an array is safe to crush.
///
/// Mirrors `CrushabilityAnalysis` at `smart_crusher.py:833-860`. The key
/// invariant: **if we don't have a reliable signal to determine which
/// items are important, we don't crush at all**. Signals include score
/// fields, error keywords, numeric anomalies, and low uniqueness.
#[derive(Debug, Clone)]
pub struct CrushabilityAnalysis {
pub crushable: bool,
pub confidence: f64,
pub reason: String,
pub signals_present: Vec<String>,
pub signals_absent: Vec<String>,
// Detailed metrics (mirroring Python field-by-field)
pub has_id_field: bool,
pub id_uniqueness: f64,
pub avg_string_uniqueness: f64,
pub has_score_field: bool,
pub error_item_count: usize,
pub anomaly_count: usize,
}
impl CrushabilityAnalysis {
/// Helper to build a "not crushable" verdict — used in several early
/// exits in `analyze_crushability`. Mirrors the Python pattern where
/// `crushable=False` paths don't bother filling in detail metrics.
pub fn skip(reason: impl Into<String>, confidence: f64) -> Self {
CrushabilityAnalysis {
crushable: false,
confidence,
reason: reason.into(),
signals_present: Vec::new(),
signals_absent: Vec::new(),
has_id_field: false,
id_uniqueness: 0.0,
avg_string_uniqueness: 0.0,
has_score_field: false,
error_item_count: 0,
anomaly_count: 0,
}
}
}
/// Complete analysis of an array.
///
/// Mirrors `ArrayAnalysis` at `smart_crusher.py:887-897`. `field_stats`
/// uses `BTreeMap` for deterministic iteration order (Python's `dict`
/// preserves insertion order; `BTreeMap` gives us a stable sorted-by-key
/// order, which is fine for the parity fixtures because the analyzer
/// builds the map by iterating sorted keys).
#[derive(Debug, Clone)]
pub struct ArrayAnalysis {
pub item_count: usize,
pub field_stats: BTreeMap<String, FieldStats>,
/// One of: `"time_series"`, `"logs"`, `"search_results"`, `"generic"`.
pub detected_pattern: String,
pub recommended_strategy: CompressionStrategy,
pub constant_fields: BTreeMap<String, Value>,
pub estimated_reduction: f64,
pub crushability: Option<CrushabilityAnalysis>,
}
/// Plan for how to compress an array.
///
/// Mirrors `CompressionPlan` at `smart_crusher.py:900-910`. `keep_indices`
/// is the list of original-array indices that survive compression;
/// `summary_ranges` carries `(start, end, summary_dict)` for runs we
/// summarized rather than dropped (currently unused in the Python impl
/// but plumbed through for parity with the dataclass).
#[derive(Debug, Clone)]
pub struct CompressionPlan {
pub strategy: CompressionStrategy,
pub keep_indices: Vec<usize>,
pub constant_fields: BTreeMap<String, Value>,
/// `(start, end, summary)` triples for summarized runs. Python uses
/// `list[tuple[int, int, dict]]`; we use `Value` for the summary so
/// any JSON shape is representable.
pub summary_ranges: Vec<(usize, usize, Value)>,
pub cluster_field: Option<String>,
pub sort_field: Option<String>,
pub keep_count: usize,
}
impl Default for CompressionPlan {
fn default() -> Self {
// Mirrors Python's @dataclass defaults at line 900-910.
CompressionPlan {
strategy: CompressionStrategy::None,
keep_indices: Vec::new(),
constant_fields: BTreeMap::new(),
summary_ranges: Vec::new(),
cluster_field: None,
sort_field: None,
keep_count: 10,
}
}
}
/// Result from `SmartCrusher.crush()` — used by ContentRouter when
/// routing JSON arrays. Mirrors `CrushResult` at `smart_crusher.py:913-923`.
#[derive(Debug, Clone)]
pub struct CrushResult {
pub compressed: String,
pub original: String,
pub was_modified: bool,
pub strategy: String,
}
impl CrushResult {
/// Pass-through result: same as input, no modification, strategy
/// `"passthrough"`. Used when content can't be compressed (not JSON,
/// too small, no crushable arrays, etc.).
pub fn passthrough(content: impl Into<String>) -> Self {
let s = content.into();
CrushResult {
compressed: s.clone(),
original: s,
was_modified: false,
strategy: "passthrough".to_string(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn compression_strategy_strings_match_python() {
// Strategy debug strings appear in the parity fixtures; these must
// not drift. If a value here changes, every fixture breaks.
assert_eq!(CompressionStrategy::None.as_str(), "none");
assert_eq!(CompressionStrategy::Skip.as_str(), "skip");
assert_eq!(CompressionStrategy::TimeSeries.as_str(), "time_series");
assert_eq!(CompressionStrategy::ClusterSample.as_str(), "cluster");
assert_eq!(CompressionStrategy::TopN.as_str(), "top_n");
assert_eq!(CompressionStrategy::SmartSample.as_str(), "smart_sample");
}
#[test]
fn crushability_skip_helper() {
let r = CrushabilityAnalysis::skip("too small", 1.0);
assert!(!r.crushable);
assert_eq!(r.confidence, 1.0);
assert_eq!(r.reason, "too small");
}
#[test]
fn compression_plan_default_keep_count_matches_python() {
// Python's @dataclass default is `keep_count: int = 10`.
let p = CompressionPlan::default();
assert_eq!(p.keep_count, 10);
assert_eq!(p.strategy, CompressionStrategy::None);
assert!(p.keep_indices.is_empty());
}
#[test]
fn crush_result_passthrough() {
let r = CrushResult::passthrough("hello");
assert_eq!(r.compressed, "hello");
assert_eq!(r.original, "hello");
assert!(!r.was_modified);
assert_eq!(r.strategy, "passthrough");
}
}

2
plugins/headroom-agent-hooks/.claude-plugin/plugin.json
View file

@ -1,6 +1,6 @@
{
"name": "headroom",
"version": "0.11.0",
"version": "0.10.17",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"author": {
"name": "Headroom Contributors",

2
plugins/headroom-agent-hooks/.github/plugin/plugin.json vendored
View file

@ -1,6 +1,6 @@
{
"name": "headroom",
"version": "0.11.0",
"version": "0.10.17",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"author": {
"name": "Headroom Contributors",