From 43d1aa0329a220ba50443a832d84b8a0c101585d Mon Sep 17 00:00:00 2001 From: chopratejas Date: Mon, 27 Apr 2026 00:01:26 -0700 Subject: [PATCH] =?UTF-8?q?parity(smart=5Fcrusher):=20byte-equal=20harness?= =?UTF-8?q?=20=E2=80=94=2017=20fixtures=20green?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the SmartCrusher half of the Rust-vs-Python parity harness. Path A from the Stage 3c.1 plan: record fixtures from Python (with real fastembed embeddings + the post-bug-fix code), drive the Rust port over the same inputs, and assert byte-equal output on every recorded scenario. What's in: - `tests/parity/record_smart_crusher.py`: standalone recorder for `SmartCrusher.crush(content, query, bias)`. The generic recorder framework only captures one positional, so this script writes its own JSON envelope `{input: {content, query, bias}, config, output}`. 17 scenarios cover the planning paths exercised by ContentRouter in production: passthrough, smart_sample, top_n, time-series, duplicates, unicode (`ensure_ascii=False`), nested-3-deep, empties, bias above and below 1.0. - `crates/headroom-parity/src/lib.rs`: `SmartCrusherComparator` reconstructs `SmartCrusherConfig` from the fixture's config block, runs Rust `SmartCrusher::crush()`, emits the same JSON shape Python serialized. - `crates/headroom-parity/examples/diff_fixture.rs`: diagnostic CLI that prints expected-vs-actual for one fixture (used during the iteration that found the serializer bug below). Serializer fix — found by the harness: SmartCrusher uses `safe_json_dumps` (compact `(",", ":")` separators + `ensure_ascii=False`) for the wire bytes. The Rust port was using `python_json_dumps` (default Python: `(", ", ": ")` + `ensure_ascii= True`), which is the right choice for hashing but wrong for the output. Refactored `anchor_selector.rs` to take a small `JsonFmt { sort_keys, compact, ensure_ascii }` config so the three flavors share one writer, added `python_safe_json_dumps`, and switched `_smart_crush_content` to call it. All three flavors now have byte-exact tests. Two unit tests in `crusher.rs` were pinning the old (wrong) format and have been re-pinned to the compact form. Cross-language status: - All 17 empty-query fixtures: byte-equal. - Embedding-driven (non-empty-query) fixtures deferred until the ~0.0002 numeric drift between Python `onnxruntime` and Rust `ort` is resolved (or until we accept the drift via a tolerance — none of the 17 fixtures exercises a borderline relevance_threshold call, and downstream code only branches at the 0.3 threshold). Tests: cargo test --workspace (388 + supporting) green. --- .../src/transforms/anchor_selector.rs | 105 ++++++--- .../src/transforms/smart_crusher/crusher.rs | 33 ++- .../headroom-parity/examples/diff_fixture.rs | 62 +++++ crates/headroom-parity/src/lib.rs | 124 ++++++++++ ...ict_array_100_sequential_b91d1fff5ba3.json | 34 +++ .../dict_array_30_b0ba1d26fd03.json | 34 +++ .../dict_array_30_bias_high_dc3b36e60560.json | 34 +++ .../dict_array_30_bias_low_a850512901bd.json | 34 +++ .../duplicate_dicts_40_40a0670dec17.json | 34 +++ .../empty_array_41ecf20098c5.json | 34 +++ .../mixed_array_3f929019b62f.json | 34 +++ .../nested_3deep_with_array_bb5036c153aa.json | 34 +++ ...nested_object_with_array_b66d43299ea1.json | 34 +++ .../non_json_passthrough_4322abf677f3.json | 34 +++ .../nulls_and_bools_06d6fd6adcab.json | 34 +++ ...ber_array_40_changepoint_8cd416c24d53.json | 34 +++ .../short_array_passthrough_aeb16224b61f.json | 34 +++ ...small_object_passthrough_801afa36a32c.json | 34 +++ .../string_array_25_6fe4a307de81.json | 34 +++ .../time_series_50_25cd28df5a50.json | 34 +++ .../unicode_dict_array_4820ed6c0c9a.json | 34 +++ tests/parity/record_smart_crusher.py | 212 ++++++++++++++++++ 22 files changed, 1075 insertions(+), 39 deletions(-) create mode 100644 crates/headroom-parity/examples/diff_fixture.rs create mode 100644 tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json create mode 100644 tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json create mode 100644 tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json create mode 100644 tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json create mode 100644 tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json create mode 100644 tests/parity/fixtures/smart_crusher/empty_array_41ecf20098c5.json create mode 100644 tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json create mode 100644 tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json create mode 100644 tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json create mode 100644 tests/parity/fixtures/smart_crusher/non_json_passthrough_4322abf677f3.json create mode 100644 tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json create mode 100644 tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json create mode 100644 tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json create mode 100644 tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json create mode 100644 tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json create mode 100644 tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json create mode 100644 tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json create mode 100644 tests/parity/record_smart_crusher.py diff --git a/crates/headroom-core/src/transforms/anchor_selector.rs b/crates/headroom-core/src/transforms/anchor_selector.rs index e6235bff6..ff6dbfc20 100644 --- a/crates/headroom-core/src/transforms/anchor_selector.rs +++ b/crates/headroom-core/src/transforms/anchor_selector.rs @@ -354,6 +354,17 @@ pub fn compute_item_hash(item: &Value) -> String { hex[..16].to_string() } +/// Python json.dumps formatting flags used by the writer below. +#[derive(Clone, Copy)] +struct JsonFmt { + /// `sort_keys=True` → alphabetical object key order. + sort_keys: bool, + /// Compact separators `(",", ":")`. False → Python default `(", ", ": ")`. + compact: bool, + /// `ensure_ascii=True` → non-ASCII becomes `\uXXXX`. False → emit UTF-8. + ensure_ascii: bool, +} + /// Python `json.dumps(value, sort_keys=True)` — exact format parity. /// /// Differences from `serde_json::to_string`: @@ -368,7 +379,15 @@ pub fn compute_item_hash(item: &Value) -> String { /// impossible so we don't handle them here. pub fn python_json_dumps_sort_keys(value: &Value) -> String { let mut out = String::new(); - write_python_json_inner(value, &mut out, true); + write_python_json_inner( + value, + &mut out, + JsonFmt { + sort_keys: true, + compact: false, + ensure_ascii: true, + }, + ); out } @@ -376,57 +395,80 @@ pub fn python_json_dumps_sort_keys(value: &Value) -> String { /// object-key insertion order (matches the JSON parser's order via /// serde_json's `preserve_order` feature). /// -/// Used by `_smart_crush_content` to re-serialize crushed JSON for -/// the proxy. Bytes differ from `to_string` because of the `, ` / -/// `: ` separators and `\uXXXX` non-ASCII escapes — both Python -/// defaults that affect output bytes the proxy may compare against. +/// Bytes differ from `to_string` because of the `, ` / `: ` separators +/// and `\uXXXX` non-ASCII escapes — both Python defaults. pub fn python_json_dumps(value: &Value) -> String { let mut out = String::new(); - write_python_json_inner(value, &mut out, false); + write_python_json_inner( + value, + &mut out, + JsonFmt { + sort_keys: false, + compact: false, + ensure_ascii: true, + }, + ); out } -fn write_python_json_inner(value: &Value, out: &mut String, sort_keys: bool) { +/// Python `safe_json_dumps(value)` — compact separators `(",", ":")` + +/// `ensure_ascii=False`, preserving object-key insertion order. This is +/// the format `SmartCrusher._smart_crush_content` uses to re-serialize +/// crushed output, so the proxy's wire bytes match Python's exactly. +pub fn python_safe_json_dumps(value: &Value) -> String { + let mut out = String::new(); + write_python_json_inner( + value, + &mut out, + JsonFmt { + sort_keys: false, + compact: true, + ensure_ascii: false, + }, + ); + out +} + +fn write_python_json_inner(value: &Value, out: &mut String, fmt: JsonFmt) { + let item_sep = if fmt.compact { "," } else { ", " }; + let kv_sep = if fmt.compact { ":" } else { ": " }; match value { Value::Null => out.push_str("null"), Value::Bool(true) => out.push_str("true"), Value::Bool(false) => out.push_str("false"), Value::Number(n) => out.push_str(&n.to_string()), - Value::String(s) => write_python_json_string(s, out), + Value::String(s) => write_python_json_string(s, out, fmt.ensure_ascii), Value::Array(arr) => { out.push('['); for (i, v) in arr.iter().enumerate() { if i > 0 { - out.push_str(", "); + out.push_str(item_sep); } - write_python_json_inner(v, out, sort_keys); + write_python_json_inner(v, out, fmt); } out.push(']'); } Value::Object(map) => { out.push('{'); - if sort_keys { - // Python's sort_keys=True → alphabetical key order. + if fmt.sort_keys { let mut keys: Vec<&String> = map.keys().collect(); keys.sort(); for (i, key) in keys.iter().enumerate() { if i > 0 { - out.push_str(", "); + out.push_str(item_sep); } - write_python_json_string(key, out); - out.push_str(": "); - write_python_json_inner(&map[key.as_str()], out, sort_keys); + write_python_json_string(key, out, fmt.ensure_ascii); + out.push_str(kv_sep); + write_python_json_inner(&map[key.as_str()], out, fmt); } } else { - // Insertion order — serde_json's preserve_order feature - // ensures Map iteration matches the source JSON. for (i, (key, val)) in map.iter().enumerate() { if i > 0 { - out.push_str(", "); + out.push_str(item_sep); } - write_python_json_string(key, out); - out.push_str(": "); - write_python_json_inner(val, out, sort_keys); + write_python_json_string(key, out, fmt.ensure_ascii); + out.push_str(kv_sep); + write_python_json_inner(val, out, fmt); } } out.push('}'); @@ -434,13 +476,18 @@ fn write_python_json_inner(value: &Value, out: &mut String, sort_keys: bool) { } } -/// Encode a string value the way Python's json.dumps does by default -/// (`ensure_ascii=True`): +/// Encode a string value Python-style. +/// +/// `ensure_ascii=true`: /// - Backslash, quote, control chars → standard escapes (`\\`, `\"`, /// `\n`, etc.). /// - Non-ASCII codepoints → `\uXXXX` (surrogate-paired for codepoints /// above 0xFFFF). -fn write_python_json_string(s: &str, out: &mut String) { +/// +/// `ensure_ascii=false`: +/// - Same standard escapes for backslash/quote/controls. +/// - Non-ASCII codepoints emit literal UTF-8 bytes. +fn write_python_json_string(s: &str, out: &mut String, ensure_ascii: bool) { out.push('"'); for c in s.chars() { match c { @@ -455,9 +502,13 @@ fn write_python_json_string(s: &str, out: &mut String) { out.push_str(&format!("\\u{:04x}", c as u32)); } c if (c as u32) <= 0x7E => out.push(c), + c if !ensure_ascii => { + // ensure_ascii=False: emit raw UTF-8 like Python does. + out.push(c); + } c => { - // Non-ASCII: encode as \uXXXX, with surrogate pair for - // codepoints above 0xFFFF. + // ensure_ascii=True: encode as \uXXXX, surrogate pair + // for codepoints above 0xFFFF. let cp = c as u32; if cp <= 0xFFFF { out.push_str(&format!("\\u{:04x}", cp)); diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs index 5de95c505..fa8ded62d 100644 --- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs @@ -171,10 +171,11 @@ impl SmartCrusher { let (crushed, info) = self.process_value(&parsed, 0, query_context, bias); - // Re-serialize with Python-compatible formatting (preserves - // insertion order; uses `, ` / `: ` separators and ASCII - // escapes for non-ASCII codepoints). - let result = crate::transforms::anchor_selector::python_json_dumps(&crushed); + // Re-serialize with Python `safe_json_dumps` formatting: + // compact `(",", ":")` separators + `ensure_ascii=False`, + // preserving object-key insertion order. Matches the Python + // SmartCrusher output bytes the proxy writes. + let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed); let was_modified = result != content.trim(); (result, was_modified, info) } @@ -764,9 +765,15 @@ mod tests { #[test] fn crush_small_array_passes_through() { let c = crusher(); - let result = c.crush(r#"[1, 2, 3]"#, "", 1.0); - // Below min_items_to_analyze=5 → no crushing. + // Compact-form input matches the compact serializer output, so + // the array is not "modified" even though it round-trips + // through parse → serialize. (The spaced form `[1, 2, 3]` + // would mark `was_modified=true` because the compact + // serializer rewrites it to `[1,2,3]`.) + let result = c.crush(r#"[1,2,3]"#, "", 1.0); + // Below min_items_to_analyze=5 → no crushing of the structure. assert!(!result.was_modified); + assert_eq!(result.compressed, "[1,2,3]"); } #[test] @@ -790,15 +797,17 @@ mod tests { } #[test] - fn crush_serializes_with_python_format() { + fn crush_serializes_with_python_safe_format() { let c = crusher(); - // 3-key object should round-trip as `{"a": 1, "b": 2, "c": 3}` - // (with spaces) — Python's default json.dumps format. - let input = r#"{"a":1,"b":2,"c":3}"#; + // SmartCrusher uses Python's `safe_json_dumps`: compact + // separators `(",", ":")` + `ensure_ascii=False`, preserving + // object-key insertion order. A spaced input round-trips to + // the compact form. + let input = r#"{"a": 1, "b": 2, "c": 3}"#; let result = c.crush(input, "", 1.0); assert_eq!( - result.compressed, r#"{"a": 1, "b": 2, "c": 3}"#, - "Python-format serializer adds spaces after `,` and `:`" + result.compressed, r#"{"a":1,"b":2,"c":3}"#, + "safe_json_dumps emits compact `,` / `:` separators" ); } diff --git a/crates/headroom-parity/examples/diff_fixture.rs b/crates/headroom-parity/examples/diff_fixture.rs new file mode 100644 index 000000000..18a80906c --- /dev/null +++ b/crates/headroom-parity/examples/diff_fixture.rs @@ -0,0 +1,62 @@ +//! Diagnostic: load one parity fixture and print expected vs actual. +//! +//! Usage: cargo run -p headroom-parity --example diff_fixture -- + +use anyhow::{bail, Context, Result}; +use headroom_parity::{builtin_comparators, Fixture}; +use std::env; +use std::fs; + +fn main() -> Result<()> { + let path = env::args() + .nth(1) + .context("usage: diff_fixture ")?; + let bytes = fs::read(&path).context("reading fixture")?; + let fixture: Fixture = serde_json::from_slice(&bytes).context("parsing fixture")?; + + let comparator = builtin_comparators() + .into_iter() + .find(|c| c.name() == fixture.transform) + .with_context(|| format!("no comparator named {}", fixture.transform))?; + + let actual = match comparator.run(&fixture.input, &fixture.config) { + Ok(v) => v, + Err(e) => bail!("comparator failed: {e}"), + }; + + let expected_pretty = serde_json::to_string_pretty(&fixture.output)?; + let actual_pretty = serde_json::to_string_pretty(&actual)?; + + println!("=== Expected (Python) ==="); + println!("{expected_pretty}"); + println!("\n=== Actual (Rust) ==="); + println!("{actual_pretty}"); + + if actual == fixture.output { + println!("\n=== MATCH ==="); + } else { + println!("\n=== DIFFER ==="); + // Field-by-field for objects + if let (Some(exp_obj), Some(act_obj)) = + (fixture.output.as_object(), actual.as_object()) + { + for key in exp_obj.keys().chain(act_obj.keys()).collect::>() { + let e = exp_obj.get(key); + let a = act_obj.get(key); + if e != a { + println!(" field {key}:"); + println!( + " expected: {}", + e.map(|v| serde_json::to_string(v).unwrap()).unwrap_or_default() + ); + println!( + " actual : {}", + a.map(|v| serde_json::to_string(v).unwrap()).unwrap_or_default() + ); + } + } + } + } + + Ok(()) +} diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs index 43720d32f..8a19a9230 100644 --- a/crates/headroom-parity/src/lib.rs +++ b/crates/headroom-parity/src/lib.rs @@ -274,6 +274,129 @@ impl TransformComparator for TokenizerComparator { } } +/// Real comparator for the `smart_crusher` transform. Drives the Rust +/// port over the recorded fixture inputs (`{content, query, bias}`) +/// and emits the same shape the Python recorder serialized: +/// `{compressed, original, was_modified, strategy}`. +/// +/// The comparator builds `SmartCrusherConfig` from the fixture's +/// `config` block, falling back to the Rust default for any missing +/// field. The Python recorder writes every field today, but tolerating +/// partial configs keeps fixtures forward-compatible if either side +/// gains a field. +pub struct SmartCrusherComparator; + +impl TransformComparator for SmartCrusherComparator { + fn name(&self) -> &str { + "smart_crusher" + } + + fn run( + &self, + input: &serde_json::Value, + config: &serde_json::Value, + ) -> Result { + use headroom_core::transforms::smart_crusher::{SmartCrusher, SmartCrusherConfig}; + + let content = input + .get("content") + .and_then(|v| v.as_str()) + .context("smart_crusher fixture input.content must be a JSON string")?; + let query = input + .get("query") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let bias = input + .get("bias") + .and_then(|v| v.as_f64()) + .unwrap_or(1.0); + + let defaults = SmartCrusherConfig::default(); + let cfg = SmartCrusherConfig { + enabled: config + .get("enabled") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.enabled), + min_items_to_analyze: config + .get("min_items_to_analyze") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + .unwrap_or(defaults.min_items_to_analyze), + min_tokens_to_crush: config + .get("min_tokens_to_crush") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + .unwrap_or(defaults.min_tokens_to_crush), + variance_threshold: config + .get("variance_threshold") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.variance_threshold), + uniqueness_threshold: config + .get("uniqueness_threshold") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.uniqueness_threshold), + similarity_threshold: config + .get("similarity_threshold") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.similarity_threshold), + max_items_after_crush: config + .get("max_items_after_crush") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + .unwrap_or(defaults.max_items_after_crush), + preserve_change_points: config + .get("preserve_change_points") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.preserve_change_points), + factor_out_constants: config + .get("factor_out_constants") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.factor_out_constants), + include_summaries: config + .get("include_summaries") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.include_summaries), + use_feedback_hints: config + .get("use_feedback_hints") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.use_feedback_hints), + toin_confidence_threshold: config + .get("toin_confidence_threshold") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.toin_confidence_threshold), + dedup_identical_items: config + .get("dedup_identical_items") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.dedup_identical_items), + first_fraction: config + .get("first_fraction") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.first_fraction), + last_fraction: config + .get("last_fraction") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.last_fraction), + // Rust-only knob; Python config has no field for it. Use + // the Rust default (which mirrors Python's hardcoded + // RelevanceConfig.relevance_threshold = 0.3). + relevance_threshold: config + .get("relevance_threshold") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.relevance_threshold), + }; + + let crusher = SmartCrusher::new(cfg); + let result = crusher.crush(content, query, bias); + + Ok(serde_json::json!({ + "compressed": result.compressed, + "original": result.original, + "was_modified": result.was_modified, + "strategy": result.strategy, + })) + } +} + /// Every built-in comparator, in a stable order. pub fn builtin_comparators() -> Vec> { vec![ @@ -282,6 +405,7 @@ pub fn builtin_comparators() -> Vec> { Box::new(CacheAlignerComparator), Box::new(TokenizerComparator), Box::new(CcrComparator), + Box::new(SmartCrusherComparator), ] } diff --git a/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json b/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json new file mode 100644 index 000000000..7810da620 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[{\"id\": 0, \"level\": \"warn\", \"message\": \"seq 0\"}, {\"id\": 1, \"level\": \"info\", \"message\": \"seq 1\"}, {\"id\": 2, \"level\": \"info\", \"message\": \"seq 2\"}, {\"id\": 3, \"level\": \"info\", \"message\": \"seq 3\"}, {\"id\": 4, \"level\": \"info\", \"message\": \"seq 4\"}, {\"id\": 5, \"level\": \"info\", \"message\": \"seq 5\"}, {\"id\": 6, \"level\": \"info\", \"message\": \"seq 6\"}, {\"id\": 7, \"level\": \"warn\", \"message\": \"seq 7\"}, {\"id\": 8, \"level\": \"info\", \"message\": \"seq 8\"}, {\"id\": 9, \"level\": \"info\", \"message\": \"seq 9\"}, {\"id\": 10, \"level\": \"info\", \"message\": \"seq 10\"}, {\"id\": 11, \"level\": \"info\", \"message\": \"seq 11\"}, {\"id\": 12, \"level\": \"info\", \"message\": \"seq 12\"}, {\"id\": 13, \"level\": \"info\", \"message\": \"seq 13\"}, {\"id\": 14, \"level\": \"warn\", \"message\": \"seq 14\"}, {\"id\": 15, \"level\": \"info\", \"message\": \"seq 15\"}, {\"id\": 16, \"level\": \"info\", \"message\": \"seq 16\"}, {\"id\": 17, \"level\": \"info\", \"message\": \"seq 17\"}, {\"id\": 18, \"level\": \"info\", \"message\": \"seq 18\"}, {\"id\": 19, \"level\": \"info\", \"message\": \"seq 19\"}, {\"id\": 20, \"level\": \"info\", \"message\": \"seq 20\"}, {\"id\": 21, \"level\": \"warn\", \"message\": \"seq 21\"}, {\"id\": 22, \"level\": \"info\", \"message\": \"seq 22\"}, {\"id\": 23, \"level\": \"info\", \"message\": \"seq 23\"}, {\"id\": 24, \"level\": \"info\", \"message\": \"seq 24\"}, {\"id\": 25, \"level\": \"info\", \"message\": \"seq 25\"}, {\"id\": 26, \"level\": \"info\", \"message\": \"seq 26\"}, {\"id\": 27, \"level\": \"info\", \"message\": \"seq 27\"}, {\"id\": 28, \"level\": \"warn\", \"message\": \"seq 28\"}, {\"id\": 29, \"level\": \"info\", \"message\": \"seq 29\"}, {\"id\": 30, \"level\": \"info\", \"message\": \"seq 30\"}, {\"id\": 31, \"level\": \"info\", \"message\": \"seq 31\"}, {\"id\": 32, \"level\": \"info\", \"message\": \"seq 32\"}, {\"id\": 33, \"level\": \"info\", \"message\": \"seq 33\"}, {\"id\": 34, \"level\": \"info\", \"message\": \"seq 34\"}, {\"id\": 35, \"level\": \"warn\", \"message\": \"seq 35\"}, {\"id\": 36, \"level\": \"info\", \"message\": \"seq 36\"}, {\"id\": 37, \"level\": \"info\", \"message\": \"seq 37\"}, {\"id\": 38, \"level\": \"info\", \"message\": \"seq 38\"}, {\"id\": 39, \"level\": \"info\", \"message\": \"seq 39\"}, {\"id\": 40, \"level\": \"info\", \"message\": \"seq 40\"}, {\"id\": 41, \"level\": \"info\", \"message\": \"seq 41\"}, {\"id\": 42, \"level\": \"warn\", \"message\": \"seq 42\"}, {\"id\": 43, \"level\": \"info\", \"message\": \"seq 43\"}, {\"id\": 44, \"level\": \"info\", \"message\": \"seq 44\"}, {\"id\": 45, \"level\": \"info\", \"message\": \"seq 45\"}, {\"id\": 46, \"level\": \"info\", \"message\": \"seq 46\"}, {\"id\": 47, \"level\": \"info\", \"message\": \"seq 47\"}, {\"id\": 48, \"level\": \"info\", \"message\": \"seq 48\"}, {\"id\": 49, \"level\": \"warn\", \"message\": \"seq 49\"}, {\"id\": 50, \"level\": \"info\", \"message\": \"seq 50\"}, {\"id\": 51, \"level\": \"info\", \"message\": \"seq 51\"}, {\"id\": 52, \"level\": \"info\", \"message\": \"seq 52\"}, {\"id\": 53, \"level\": \"info\", \"message\": \"seq 53\"}, {\"id\": 54, \"level\": \"info\", \"message\": \"seq 54\"}, {\"id\": 55, \"level\": \"info\", \"message\": \"seq 55\"}, {\"id\": 56, \"level\": \"warn\", \"message\": \"seq 56\"}, {\"id\": 57, \"level\": \"info\", \"message\": \"seq 57\"}, {\"id\": 58, \"level\": \"info\", \"message\": \"seq 58\"}, {\"id\": 59, \"level\": \"info\", \"message\": \"seq 59\"}, {\"id\": 60, \"level\": \"info\", \"message\": \"seq 60\"}, {\"id\": 61, \"level\": \"info\", \"message\": \"seq 61\"}, {\"id\": 62, \"level\": \"info\", \"message\": \"seq 62\"}, {\"id\": 63, \"level\": \"warn\", \"message\": \"seq 63\"}, {\"id\": 64, \"level\": \"info\", \"message\": \"seq 64\"}, {\"id\": 65, \"level\": \"info\", \"message\": \"seq 65\"}, {\"id\": 66, \"level\": \"info\", \"message\": \"seq 66\"}, {\"id\": 67, \"level\": \"info\", \"message\": \"seq 67\"}, {\"id\": 68, \"level\": \"info\", \"message\": \"seq 68\"}, {\"id\": 69, \"level\": \"info\", \"message\": \"seq 69\"}, {\"id\": 70, \"level\": \"warn\", \"message\": \"seq 70\"}, {\"id\": 71, \"level\": \"info\", \"message\": \"seq 71\"}, {\"id\": 72, \"level\": \"info\", \"message\": \"seq 72\"}, {\"id\": 73, \"level\": \"info\", \"message\": \"seq 73\"}, {\"id\": 74, \"level\": \"info\", \"message\": \"seq 74\"}, {\"id\": 75, \"level\": \"info\", \"message\": \"seq 75\"}, {\"id\": 76, \"level\": \"info\", \"message\": \"seq 76\"}, {\"id\": 77, \"level\": \"warn\", \"message\": \"seq 77\"}, {\"id\": 78, \"level\": \"info\", \"message\": \"seq 78\"}, {\"id\": 79, \"level\": \"info\", \"message\": \"seq 79\"}, {\"id\": 80, \"level\": \"info\", \"message\": \"seq 80\"}, {\"id\": 81, \"level\": \"info\", \"message\": \"seq 81\"}, {\"id\": 82, \"level\": \"info\", \"message\": \"seq 82\"}, {\"id\": 83, \"level\": \"info\", \"message\": \"seq 83\"}, {\"id\": 84, \"level\": \"warn\", \"message\": \"seq 84\"}, {\"id\": 85, \"level\": \"info\", \"message\": \"seq 85\"}, {\"id\": 86, \"level\": \"info\", \"message\": \"seq 86\"}, {\"id\": 87, \"level\": \"info\", \"message\": \"seq 87\"}, {\"id\": 88, \"level\": \"info\", \"message\": \"seq 88\"}, {\"id\": 89, \"level\": \"info\", \"message\": \"seq 89\"}, {\"id\": 90, \"level\": \"info\", \"message\": \"seq 90\"}, {\"id\": 91, \"level\": \"warn\", \"message\": \"seq 91\"}, {\"id\": 92, \"level\": \"info\", \"message\": \"seq 92\"}, {\"id\": 93, \"level\": \"info\", \"message\": \"seq 93\"}, {\"id\": 94, \"level\": \"info\", \"message\": \"seq 94\"}, {\"id\": 95, \"level\": \"info\", \"message\": \"seq 95\"}, {\"id\": 96, \"level\": \"info\", \"message\": \"seq 96\"}, {\"id\": 97, \"level\": \"info\", \"message\": \"seq 97\"}, {\"id\": 98, \"level\": \"warn\", \"message\": \"seq 98\"}, {\"id\": 99, \"level\": \"info\", \"message\": \"seq 99\"}]", + "query": "" + }, + "input_sha256": "b91d1fff5ba3ca82d797014b6f93c01e5e4ab7e2f3667c463a0015a2166617c3", + "label": "dict_array_100_sequential", + "output": { + "compressed": "[{\"id\":0,\"level\":\"warn\",\"message\":\"seq 0\"},{\"id\":7,\"level\":\"warn\",\"message\":\"seq 7\"},{\"id\":14,\"level\":\"warn\",\"message\":\"seq 14\"},{\"id\":21,\"level\":\"warn\",\"message\":\"seq 21\"},{\"id\":28,\"level\":\"warn\",\"message\":\"seq 28\"},{\"id\":35,\"level\":\"warn\",\"message\":\"seq 35\"},{\"id\":42,\"level\":\"warn\",\"message\":\"seq 42\"},{\"id\":49,\"level\":\"warn\",\"message\":\"seq 49\"},{\"id\":56,\"level\":\"warn\",\"message\":\"seq 56\"},{\"id\":63,\"level\":\"warn\",\"message\":\"seq 63\"},{\"id\":70,\"level\":\"warn\",\"message\":\"seq 70\"},{\"id\":77,\"level\":\"warn\",\"message\":\"seq 77\"},{\"id\":84,\"level\":\"warn\",\"message\":\"seq 84\"},{\"id\":91,\"level\":\"warn\",\"message\":\"seq 91\"},{\"id\":98,\"level\":\"warn\",\"message\":\"seq 98\"}]", + "original": "[{\"id\": 0, \"level\": \"warn\", \"message\": \"seq 0\"}, {\"id\": 1, \"level\": \"info\", \"message\": \"seq 1\"}, {\"id\": 2, \"level\": \"info\", \"message\": \"seq 2\"}, {\"id\": 3, \"level\": \"info\", \"message\": \"seq 3\"}, {\"id\": 4, \"level\": \"info\", \"message\": \"seq 4\"}, {\"id\": 5, \"level\": \"info\", \"message\": \"seq 5\"}, {\"id\": 6, \"level\": \"info\", \"message\": \"seq 6\"}, {\"id\": 7, \"level\": \"warn\", \"message\": \"seq 7\"}, {\"id\": 8, \"level\": \"info\", \"message\": \"seq 8\"}, {\"id\": 9, \"level\": \"info\", \"message\": \"seq 9\"}, {\"id\": 10, \"level\": \"info\", \"message\": \"seq 10\"}, {\"id\": 11, \"level\": \"info\", \"message\": \"seq 11\"}, {\"id\": 12, \"level\": \"info\", \"message\": \"seq 12\"}, {\"id\": 13, \"level\": \"info\", \"message\": \"seq 13\"}, {\"id\": 14, \"level\": \"warn\", \"message\": \"seq 14\"}, {\"id\": 15, \"level\": \"info\", \"message\": \"seq 15\"}, {\"id\": 16, \"level\": \"info\", \"message\": \"seq 16\"}, {\"id\": 17, \"level\": \"info\", \"message\": \"seq 17\"}, {\"id\": 18, \"level\": \"info\", \"message\": \"seq 18\"}, {\"id\": 19, \"level\": \"info\", \"message\": \"seq 19\"}, {\"id\": 20, \"level\": \"info\", \"message\": \"seq 20\"}, {\"id\": 21, \"level\": \"warn\", \"message\": \"seq 21\"}, {\"id\": 22, \"level\": \"info\", \"message\": \"seq 22\"}, {\"id\": 23, \"level\": \"info\", \"message\": \"seq 23\"}, {\"id\": 24, \"level\": \"info\", \"message\": \"seq 24\"}, {\"id\": 25, \"level\": \"info\", \"message\": \"seq 25\"}, {\"id\": 26, \"level\": \"info\", \"message\": \"seq 26\"}, {\"id\": 27, \"level\": \"info\", \"message\": \"seq 27\"}, {\"id\": 28, \"level\": \"warn\", \"message\": \"seq 28\"}, {\"id\": 29, \"level\": \"info\", \"message\": \"seq 29\"}, {\"id\": 30, \"level\": \"info\", \"message\": \"seq 30\"}, {\"id\": 31, \"level\": \"info\", \"message\": \"seq 31\"}, {\"id\": 32, \"level\": \"info\", \"message\": \"seq 32\"}, {\"id\": 33, \"level\": \"info\", \"message\": \"seq 33\"}, {\"id\": 34, \"level\": \"info\", \"message\": \"seq 34\"}, {\"id\": 35, \"level\": \"warn\", \"message\": \"seq 35\"}, {\"id\": 36, \"level\": \"info\", \"message\": \"seq 36\"}, {\"id\": 37, \"level\": \"info\", \"message\": \"seq 37\"}, {\"id\": 38, \"level\": \"info\", \"message\": \"seq 38\"}, {\"id\": 39, \"level\": \"info\", \"message\": \"seq 39\"}, {\"id\": 40, \"level\": \"info\", \"message\": \"seq 40\"}, {\"id\": 41, \"level\": \"info\", \"message\": \"seq 41\"}, {\"id\": 42, \"level\": \"warn\", \"message\": \"seq 42\"}, {\"id\": 43, \"level\": \"info\", \"message\": \"seq 43\"}, {\"id\": 44, \"level\": \"info\", \"message\": \"seq 44\"}, {\"id\": 45, \"level\": \"info\", \"message\": \"seq 45\"}, {\"id\": 46, \"level\": \"info\", \"message\": \"seq 46\"}, {\"id\": 47, \"level\": \"info\", \"message\": \"seq 47\"}, {\"id\": 48, \"level\": \"info\", \"message\": \"seq 48\"}, {\"id\": 49, \"level\": \"warn\", \"message\": \"seq 49\"}, {\"id\": 50, \"level\": \"info\", \"message\": \"seq 50\"}, {\"id\": 51, \"level\": \"info\", \"message\": \"seq 51\"}, {\"id\": 52, \"level\": \"info\", \"message\": \"seq 52\"}, {\"id\": 53, \"level\": \"info\", \"message\": \"seq 53\"}, {\"id\": 54, \"level\": \"info\", \"message\": \"seq 54\"}, {\"id\": 55, \"level\": \"info\", \"message\": \"seq 55\"}, {\"id\": 56, \"level\": \"warn\", \"message\": \"seq 56\"}, {\"id\": 57, \"level\": \"info\", \"message\": \"seq 57\"}, {\"id\": 58, \"level\": \"info\", \"message\": \"seq 58\"}, {\"id\": 59, \"level\": \"info\", \"message\": \"seq 59\"}, {\"id\": 60, \"level\": \"info\", \"message\": \"seq 60\"}, {\"id\": 61, \"level\": \"info\", \"message\": \"seq 61\"}, {\"id\": 62, \"level\": \"info\", \"message\": \"seq 62\"}, {\"id\": 63, \"level\": \"warn\", \"message\": \"seq 63\"}, {\"id\": 64, \"level\": \"info\", \"message\": \"seq 64\"}, {\"id\": 65, \"level\": \"info\", \"message\": \"seq 65\"}, {\"id\": 66, \"level\": \"info\", \"message\": \"seq 66\"}, {\"id\": 67, \"level\": \"info\", \"message\": \"seq 67\"}, {\"id\": 68, \"level\": \"info\", \"message\": \"seq 68\"}, {\"id\": 69, \"level\": \"info\", \"message\": \"seq 69\"}, {\"id\": 70, \"level\": \"warn\", \"message\": \"seq 70\"}, {\"id\": 71, \"level\": \"info\", \"message\": \"seq 71\"}, {\"id\": 72, \"level\": \"info\", \"message\": \"seq 72\"}, {\"id\": 73, \"level\": \"info\", \"message\": \"seq 73\"}, {\"id\": 74, \"level\": \"info\", \"message\": \"seq 74\"}, {\"id\": 75, \"level\": \"info\", \"message\": \"seq 75\"}, {\"id\": 76, \"level\": \"info\", \"message\": \"seq 76\"}, {\"id\": 77, \"level\": \"warn\", \"message\": \"seq 77\"}, {\"id\": 78, \"level\": \"info\", \"message\": \"seq 78\"}, {\"id\": 79, \"level\": \"info\", \"message\": \"seq 79\"}, {\"id\": 80, \"level\": \"info\", \"message\": \"seq 80\"}, {\"id\": 81, \"level\": \"info\", \"message\": \"seq 81\"}, {\"id\": 82, \"level\": \"info\", \"message\": \"seq 82\"}, {\"id\": 83, \"level\": \"info\", \"message\": \"seq 83\"}, {\"id\": 84, \"level\": \"warn\", \"message\": \"seq 84\"}, {\"id\": 85, \"level\": \"info\", \"message\": \"seq 85\"}, {\"id\": 86, \"level\": \"info\", \"message\": \"seq 86\"}, {\"id\": 87, \"level\": \"info\", \"message\": \"seq 87\"}, {\"id\": 88, \"level\": \"info\", \"message\": \"seq 88\"}, {\"id\": 89, \"level\": \"info\", \"message\": \"seq 89\"}, {\"id\": 90, \"level\": \"info\", \"message\": \"seq 90\"}, {\"id\": 91, \"level\": \"warn\", \"message\": \"seq 91\"}, {\"id\": 92, \"level\": \"info\", \"message\": \"seq 92\"}, {\"id\": 93, \"level\": \"info\", \"message\": \"seq 93\"}, {\"id\": 94, \"level\": \"info\", \"message\": \"seq 94\"}, {\"id\": 95, \"level\": \"info\", \"message\": \"seq 95\"}, {\"id\": 96, \"level\": \"info\", \"message\": \"seq 96\"}, {\"id\": 97, \"level\": \"info\", \"message\": \"seq 97\"}, {\"id\": 98, \"level\": \"warn\", \"message\": \"seq 98\"}, {\"id\": 99, \"level\": \"info\", \"message\": \"seq 99\"}]", + "strategy": "smart_sample(100->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:40.005010+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json b/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json new file mode 100644 index 000000000..8da36040d --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]", + "query": "" + }, + "input_sha256": "b0ba1d26fd03d9b5010ffd8d4c6185e0c5baf03ab783316f3ad15830d1bba779", + "label": "dict_array_30", + "output": { + "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]", + "original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]", + "strategy": "smart_sample(30->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.922238+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json new file mode 100644 index 000000000..9dbeb71d7 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.5, + "content": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]", + "query": "" + }, + "input_sha256": "dc3b36e605600ba3d7b91fae6a05d2fb3c849de875c455530d61ed40bbb2c956", + "label": "dict_array_30_bias_high", + "output": { + "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]", + "original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]", + "strategy": "smart_sample(30->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.946175+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json new file mode 100644 index 000000000..091b94ec6 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 0.7, + "content": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]", + "query": "" + }, + "input_sha256": "a850512901bdb42373b58e753e7a9ed2ae20fab671f686f2fd7a1c670889eb93", + "label": "dict_array_30_bias_low", + "output": { + "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]", + "original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]", + "strategy": "smart_sample(30->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.955935+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json b/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json new file mode 100644 index 000000000..e1d0e4c91 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[{\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}]", + "query": "" + }, + "input_sha256": "40a0670dec17ae0f765b640ffea2a78c2f2526c03877dcabf0b6108b608ada8a", + "label": "duplicate_dicts_40", + "output": { + "compressed": "[{\"event\":\"heartbeat\",\"ok\":true}]", + "original": "[{\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}]", + "strategy": "smart_sample(40->1)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:40.029276+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/empty_array_41ecf20098c5.json b/tests/parity/fixtures/smart_crusher/empty_array_41ecf20098c5.json new file mode 100644 index 000000000..948121ffd --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/empty_array_41ecf20098c5.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[]", + "query": "" + }, + "input_sha256": "41ecf20098c599d75649f87718e0160024927b289bf03f635d5be1dca68bfda9", + "label": "empty_array", + "output": { + "compressed": "[]", + "original": "[]", + "strategy": "passthrough", + "was_modified": false + }, + "recorded_at": "2026-04-27T06:59:40.029560+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json b/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json new file mode 100644 index 000000000..4a9f61509 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[\"start\", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, \"middle\", \"end\", \"end\", \"end\", \"end\", \"end\"]", + "query": "" + }, + "input_sha256": "3f929019b62fa97b372e2d8aad705dbef029ddb7934acdb237083fdead8162c0", + "label": "mixed_array", + "output": { + "compressed": "[\"start\",0,1,2,3,18,19,\"middle\",\"end\",\"end\",\"end\",\"end\",\"end\"]", + "original": "[\"start\", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, \"middle\", \"end\", \"end\", \"end\", \"end\", \"end\"]", + "strategy": "mixed:adaptive(27->13,str:7->7,num:20)(27->13)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.930363+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json b/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json new file mode 100644 index 000000000..3c0638562 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}", + "query": "" + }, + "input_sha256": "bb5036c153aa8a9e6700b751413952c4753c328cc8bd63986f68c3a5b4fb9cb6", + "label": "nested_3deep_with_array", + "output": { + "compressed": "{\"a\":{\"b\":{\"events\":[{\"i\":0,\"kind\":\"deep\",\"v\":\"x0\"},{\"i\":1,\"kind\":\"deep\",\"v\":\"x1\"},{\"i\":2,\"kind\":\"deep\",\"v\":\"x2\"},{\"i\":3,\"kind\":\"deep\",\"v\":\"x3\"},{\"i\":4,\"kind\":\"deep\",\"v\":\"x4\"},{\"i\":5,\"kind\":\"deep\",\"v\":\"x5\"},{\"i\":6,\"kind\":\"deep\",\"v\":\"x6\"},{\"i\":7,\"kind\":\"deep\",\"v\":\"x7\"},{\"i\":8,\"kind\":\"deep\",\"v\":\"x8\"},{\"i\":9,\"kind\":\"deep\",\"v\":\"x9\"},{\"i\":10,\"kind\":\"deep\",\"v\":\"x10\"},{\"i\":11,\"kind\":\"deep\",\"v\":\"x11\"},{\"i\":12,\"kind\":\"deep\",\"v\":\"x12\"},{\"i\":13,\"kind\":\"deep\",\"v\":\"x13\"},{\"i\":14,\"kind\":\"deep\",\"v\":\"x14\"}]}}}", + "original": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}", + "strategy": "none:adaptive_at_limit(15->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:41.857099+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json b/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json new file mode 100644 index 000000000..3c5c22c7d --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}", + "query": "" + }, + "input_sha256": "b66d43299ea15229c3fea6607b8fc697e32e72811ce4364c71bc9f64357208f1", + "label": "nested_object_with_array", + "output": { + "compressed": "{\"request_id\":\"req-1\",\"events\":[{\"step\":0,\"kind\":\"trace\",\"msg\":\"e0\"},{\"step\":1,\"kind\":\"trace\",\"msg\":\"e1\"},{\"step\":2,\"kind\":\"trace\",\"msg\":\"e2\"},{\"step\":3,\"kind\":\"trace\",\"msg\":\"e3\"},{\"step\":4,\"kind\":\"trace\",\"msg\":\"e4\"},{\"step\":5,\"kind\":\"trace\",\"msg\":\"e5\"},{\"step\":6,\"kind\":\"trace\",\"msg\":\"e6\"},{\"step\":7,\"kind\":\"trace\",\"msg\":\"e7\"},{\"step\":8,\"kind\":\"trace\",\"msg\":\"e8\"},{\"step\":9,\"kind\":\"trace\",\"msg\":\"e9\"},{\"step\":10,\"kind\":\"trace\",\"msg\":\"e10\"},{\"step\":11,\"kind\":\"trace\",\"msg\":\"e11\"},{\"step\":12,\"kind\":\"trace\",\"msg\":\"e12\"},{\"step\":13,\"kind\":\"trace\",\"msg\":\"e13\"},{\"step\":14,\"kind\":\"trace\",\"msg\":\"e14\"},{\"step\":15,\"kind\":\"trace\",\"msg\":\"e15\"},{\"step\":16,\"kind\":\"trace\",\"msg\":\"e16\"},{\"step\":17,\"kind\":\"trace\",\"msg\":\"e17\"},{\"step\":18,\"kind\":\"trace\",\"msg\":\"e18\"},{\"step\":19,\"kind\":\"trace\",\"msg\":\"e19\"}]}", + "original": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}", + "strategy": "skip:unique_entities_no_signal(20->20)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.935923+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/non_json_passthrough_4322abf677f3.json b/tests/parity/fixtures/smart_crusher/non_json_passthrough_4322abf677f3.json new file mode 100644 index 000000000..7fc3bbff0 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/non_json_passthrough_4322abf677f3.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "this is not json at all", + "query": "" + }, + "input_sha256": "4322abf677f3994fbc00cbaf53ff6fb3935075029f2e122feadc3b7525bd81af", + "label": "non_json_passthrough", + "output": { + "compressed": "this is not json at all", + "original": "this is not json at all", + "strategy": "passthrough", + "was_modified": false + }, + "recorded_at": "2026-04-27T06:59:39.894849+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json b/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json new file mode 100644 index 000000000..041adc374 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[null, true, false, null, true, false, null]", + "query": "" + }, + "input_sha256": "06d6fd6adcab251afc35de66e5cd54cfc21f15175bb082286cd1255b0f5bd965", + "label": "nulls_and_bools", + "output": { + "compressed": "[null,true,false,null,true,false,null]", + "original": "[null, true, false, null, true, false, null]", + "strategy": "mixed:passthrough(7->7)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:40.029745+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json b/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json new file mode 100644 index 000000000..2514bdf51 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69]", + "query": "" + }, + "input_sha256": "8cd416c24d53a5c61ebf4ad2e06963169de552cf163331df276773459af00275", + "label": "number_array_40_changepoint", + "output": { + "compressed": "[10,11,12,10,10,10,10,10,10,51,54,57,60,68,69]", + "original": "[10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69]", + "strategy": "number:adaptive(40->15,min=10,max=69,mean=35.23,median=31,stddev=24.94,p25=11,p75=59.25)(40->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.929783+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json b/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json new file mode 100644 index 000000000..fb2fc97d5 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[1, 2, 3]", + "query": "" + }, + "input_sha256": "aeb16224b61f335e6813afa323dc5d22c23b357fe6eb10ee2f8cfb3de6bfe4cf", + "label": "short_array_passthrough", + "output": { + "compressed": "[1,2,3]", + "original": "[1, 2, 3]", + "strategy": "passthrough", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.895735+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json b/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json new file mode 100644 index 000000000..c5c6e03b3 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}", + "query": "" + }, + "input_sha256": "801afa36a32c7ed7066e80c8a28fefedbb979918e679b5640803fb5d06ef6fbf", + "label": "small_object_passthrough", + "output": { + "compressed": "{\"a\":1,\"b\":2,\"c\":\"hello\"}", + "original": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}", + "strategy": "passthrough", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.895485+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json b/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json new file mode 100644 index 000000000..58f8122a2 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[\"event 0: something happened at index 0\", \"event 1: something happened at index 1\", \"event 2: something happened at index 2\", \"event 3: something happened at index 3\", \"event 4: something happened at index 4\", \"event 5: something happened at index 5\", \"event 6: something happened at index 6\", \"event 7: something happened at index 7\", \"event 8: something happened at index 8\", \"event 9: something happened at index 9\", \"event 10: something happened at index 10\", \"event 11: something happened at index 11\", \"event 12: something happened at index 12\", \"event 13: something happened at index 13\", \"event 14: something happened at index 14\", \"event 15: something happened at index 15\", \"event 16: something happened at index 16\", \"event 17: something happened at index 17\", \"event 18: something happened at index 18\", \"event 19: something happened at index 19\", \"event 20: something happened at index 20\", \"event 21: something happened at index 21\", \"event 22: something happened at index 22\", \"event 23: something happened at index 23\", \"event 24: something happened at index 24\"]", + "query": "" + }, + "input_sha256": "6fe4a307de817df20507df7fc88f0237c203a785c550696abb1f6b8dd413b40e", + "label": "string_array_25", + "output": { + "compressed": "[\"event 0: something happened at index 0\",\"event 1: something happened at index 1\",\"event 2: something happened at index 2\",\"event 3: something happened at index 3\",\"event 4: something happened at index 4\",\"event 6: something happened at index 6\",\"event 8: something happened at index 8\",\"event 10: something happened at index 10\",\"event 12: something happened at index 12\",\"event 14: something happened at index 14\",\"event 16: something happened at index 16\",\"event 18: something happened at index 18\",\"event 20: something happened at index 20\",\"event 23: something happened at index 23\",\"event 24: something happened at index 24\"]", + "original": "[\"event 0: something happened at index 0\", \"event 1: something happened at index 1\", \"event 2: something happened at index 2\", \"event 3: something happened at index 3\", \"event 4: something happened at index 4\", \"event 5: something happened at index 5\", \"event 6: something happened at index 6\", \"event 7: something happened at index 7\", \"event 8: something happened at index 8\", \"event 9: something happened at index 9\", \"event 10: something happened at index 10\", \"event 11: something happened at index 11\", \"event 12: something happened at index 12\", \"event 13: something happened at index 13\", \"event 14: something happened at index 14\", \"event 15: something happened at index 15\", \"event 16: something happened at index 16\", \"event 17: something happened at index 17\", \"event 18: something happened at index 18\", \"event 19: something happened at index 19\", \"event 20: something happened at index 20\", \"event 21: something happened at index 21\", \"event 22: something happened at index 22\", \"event 23: something happened at index 23\", \"event 24: something happened at index 24\"]", + "strategy": "string:adaptive(25->15)(25->15)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.928634+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json b/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json new file mode 100644 index 000000000..05622ecde --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]", + "query": "" + }, + "input_sha256": "25cd28df5a50521ac1f669482e1fe30bf2422f555ddd1a4d7506ca686dbd0734", + "label": "time_series_50", + "output": { + "compressed": "[{\"ts\":1000,\"metric\":0.0,\"host\":\"host-0\"},{\"ts\":1001,\"metric\":1.5,\"host\":\"host-1\"},{\"ts\":1002,\"metric\":3.0,\"host\":\"host-2\"},{\"ts\":1003,\"metric\":4.5,\"host\":\"host-0\"},{\"ts\":1004,\"metric\":6.0,\"host\":\"host-1\"},{\"ts\":1005,\"metric\":7.5,\"host\":\"host-2\"},{\"ts\":1006,\"metric\":9.0,\"host\":\"host-0\"},{\"ts\":1007,\"metric\":10.5,\"host\":\"host-1\"},{\"ts\":1008,\"metric\":12.0,\"host\":\"host-2\"},{\"ts\":1009,\"metric\":13.5,\"host\":\"host-0\"},{\"ts\":1010,\"metric\":15.0,\"host\":\"host-1\"},{\"ts\":1011,\"metric\":16.5,\"host\":\"host-2\"},{\"ts\":1012,\"metric\":18.0,\"host\":\"host-0\"},{\"ts\":1013,\"metric\":19.5,\"host\":\"host-1\"},{\"ts\":1014,\"metric\":21.0,\"host\":\"host-2\"},{\"ts\":1015,\"metric\":22.5,\"host\":\"host-0\"},{\"ts\":1016,\"metric\":24.0,\"host\":\"host-1\"},{\"ts\":1017,\"metric\":25.5,\"host\":\"host-2\"},{\"ts\":1018,\"metric\":27.0,\"host\":\"host-0\"},{\"ts\":1019,\"metric\":28.5,\"host\":\"host-1\"},{\"ts\":1020,\"metric\":30.0,\"host\":\"host-2\"},{\"ts\":1021,\"metric\":31.5,\"host\":\"host-0\"},{\"ts\":1022,\"metric\":33.0,\"host\":\"host-1\"},{\"ts\":1023,\"metric\":34.5,\"host\":\"host-2\"},{\"ts\":1024,\"metric\":36.0,\"host\":\"host-0\"},{\"ts\":1025,\"metric\":37.5,\"host\":\"host-1\"},{\"ts\":1026,\"metric\":39.0,\"host\":\"host-2\"},{\"ts\":1027,\"metric\":40.5,\"host\":\"host-0\"},{\"ts\":1028,\"metric\":42.0,\"host\":\"host-1\"},{\"ts\":1029,\"metric\":43.5,\"host\":\"host-2\"},{\"ts\":1030,\"metric\":45.0,\"host\":\"host-0\"},{\"ts\":1031,\"metric\":46.5,\"host\":\"host-1\"},{\"ts\":1032,\"metric\":48.0,\"host\":\"host-2\"},{\"ts\":1033,\"metric\":49.5,\"host\":\"host-0\"},{\"ts\":1034,\"metric\":51.0,\"host\":\"host-1\"},{\"ts\":1035,\"metric\":52.5,\"host\":\"host-2\"},{\"ts\":1036,\"metric\":54.0,\"host\":\"host-0\"},{\"ts\":1037,\"metric\":55.5,\"host\":\"host-1\"},{\"ts\":1038,\"metric\":57.0,\"host\":\"host-2\"},{\"ts\":1039,\"metric\":58.5,\"host\":\"host-0\"},{\"ts\":1040,\"metric\":60.0,\"host\":\"host-1\"},{\"ts\":1041,\"metric\":61.5,\"host\":\"host-2\"},{\"ts\":1042,\"metric\":63.0,\"host\":\"host-0\"},{\"ts\":1043,\"metric\":64.5,\"host\":\"host-1\"},{\"ts\":1044,\"metric\":66.0,\"host\":\"host-2\"},{\"ts\":1045,\"metric\":67.5,\"host\":\"host-0\"},{\"ts\":1046,\"metric\":69.0,\"host\":\"host-1\"},{\"ts\":1047,\"metric\":70.5,\"host\":\"host-2\"},{\"ts\":1048,\"metric\":72.0,\"host\":\"host-0\"},{\"ts\":1049,\"metric\":73.5,\"host\":\"host-1\"}]", + "original": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]", + "strategy": "skip:unique_entities_no_signal(50->50)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:40.020402+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json b/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json new file mode 100644 index 000000000..a4af718d2 --- /dev/null +++ b/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json @@ -0,0 +1,34 @@ +{ + "config": { + "dedup_identical_items": true, + "enabled": true, + "factor_out_constants": false, + "first_fraction": 0.3, + "include_summaries": false, + "last_fraction": 0.15, + "max_items_after_crush": 15, + "min_items_to_analyze": 5, + "min_tokens_to_crush": 200, + "preserve_change_points": true, + "similarity_threshold": 0.8, + "toin_confidence_threshold": 0.5, + "uniqueness_threshold": 0.1, + "use_feedback_hints": true, + "variance_threshold": 2.0 + }, + "input": { + "bias": 1.0, + "content": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]", + "query": "" + }, + "input_sha256": "4820ed6c0c9a4f52ab17d9bd0d6e7010fea978a227b24cb2a7b0cdba5f6fe5a6", + "label": "unicode_dict_array", + "output": { + "compressed": "[{\"id\":0,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 0\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":1,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 1\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":2,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 2\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":3,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 3\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":4,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 4\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":5,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 5\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":6,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 6\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":7,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 7\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":8,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 8\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":9,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 9\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":10,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 10\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":11,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 11\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":12,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 12\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":13,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 13\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":14,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 14\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":15,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 15\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":16,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 16\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":17,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 17\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":18,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 18\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":19,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 19\",\"tag\":\"\u0442\u0435\u0441\u0442\"}]", + "original": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]", + "strategy": "skip:unique_entities_no_signal(20->20)", + "was_modified": true + }, + "recorded_at": "2026-04-27T06:59:39.970946+00:00", + "transform": "smart_crusher" +} diff --git a/tests/parity/record_smart_crusher.py b/tests/parity/record_smart_crusher.py new file mode 100644 index 000000000..e8e24a7fe --- /dev/null +++ b/tests/parity/record_smart_crusher.py @@ -0,0 +1,212 @@ +"""Record SmartCrusher parity fixtures. + +Standalone recorder for `SmartCrusher.crush(content, query, bias)`. The +generic `recorder.py` only captures one positional arg; this script +captures all three so the Rust comparator gets the same inputs. + +Fixture schema (consumed by `SmartCrusherComparator` in +`crates/headroom-parity/src/lib.rs`): + +``` +{ + "transform": "smart_crusher", + "input": { "content": "", "query": "", "bias": 1.0 }, + "config": { ...SmartCrusherConfig fields... }, + "output": { "compressed": "...", "original": "...", + "was_modified": , "strategy": "..." }, + "recorded_at": "", + "input_sha256": "" +} +``` + +Initial fixture suite focuses on empty-query paths so embedding +nondeterminism between Python `onnxruntime` and Rust `ort` does not +factor in. With `query=""`, both BM25 and embedding scorers short- +circuit to 0.0, no items get pinned by relevance, and the output is a +deterministic function of the input. + +Run from repo root: + python tests/parity/record_smart_crusher.py +""" + +from __future__ import annotations + +import datetime as _dt +import hashlib +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +_FIXTURES_DIR = _REPO_ROOT / "tests" / "parity" / "fixtures" / "smart_crusher" + + +def _digest(payload: dict[str, Any]) -> str: + blob = json.dumps(payload, sort_keys=True).encode("utf-8") + return hashlib.sha256(blob).hexdigest() + + +def _record( + label: str, + content: str, + query: str, + bias: float, + config: SmartCrusherConfig | None = None, +) -> Path: + cfg = config or SmartCrusherConfig() + crusher = SmartCrusher(config=cfg) + result = crusher.crush(content, query=query, bias=bias) + + payload_input = {"content": content, "query": query, "bias": bias} + payload_config = asdict(cfg) + payload_output = { + "compressed": result.compressed, + "original": result.original, + "was_modified": result.was_modified, + "strategy": result.strategy, + } + + digest_source = { + "transform": "smart_crusher", + "label": label, + "input": payload_input, + "config": payload_config, + } + digest = _digest(digest_source) + + fixture = { + "transform": "smart_crusher", + "label": label, + "input": payload_input, + "config": payload_config, + "output": payload_output, + "recorded_at": _dt.datetime.now(tz=_dt.timezone.utc).isoformat(), + "input_sha256": digest, + } + + _FIXTURES_DIR.mkdir(parents=True, exist_ok=True) + target = _FIXTURES_DIR / f"{label}_{digest[:12]}.json" + target.write_text(json.dumps(fixture, indent=2, sort_keys=True) + "\n") + return target + + +def _scenarios() -> list[tuple[str, str, str, float]]: + """Initial parity scenarios. All use `query=""` to keep embeddings + out of the comparison until we resolve the ~0.0002 numeric drift + between Python `onnxruntime` and Rust `ort`.""" + out: list[tuple[str, str, str, float]] = [] + + # 1. Non-JSON content → passthrough. The crusher returns the input + # unchanged; trivially byte-equal. + out.append(("non_json_passthrough", "this is not json at all", "", 1.0)) + + # 2. JSON object with no array fields long enough to crush. + out.append( + ( + "small_object_passthrough", + json.dumps({"a": 1, "b": 2, "c": "hello"}), + "", + 1.0, + ) + ) + + # 3. Short array (below min_items_to_analyze=5) → passthrough. + out.append( + ( + "short_array_passthrough", + json.dumps([1, 2, 3]), + "", + 1.0, + ) + ) + + # 4. Dict array with 30 items, varied integer status field. + # Exercises crush_array's adaptive_k → smart_sample / top_n path. + items_30_dict = [ + {"id": i, "status": "ok" if i % 5 != 0 else "error", "msg": f"line {i}"} for i in range(30) + ] + out.append(("dict_array_30", json.dumps(items_30_dict), "", 1.0)) + + # 5. Pure string array of 25 items. + string_arr_25 = [f"event {i}: something happened at index {i}" for i in range(25)] + out.append(("string_array_25", json.dumps(string_arr_25), "", 1.0)) + + # 6. Pure number array of 40 items with a clear change point. + number_arr_40 = [10 + (i % 3) for i in range(20)] + [50 + i for i in range(20)] + out.append(("number_array_40_changepoint", json.dumps(number_arr_40), "", 1.0)) + + # 7. Mixed array (strings + ints). + mixed_arr = ["start"] + list(range(20)) + ["middle"] + ["end"] * 5 + out.append(("mixed_array", json.dumps(mixed_arr), "", 1.0)) + + # 8. Nested: top-level dict whose `events` field is a long dict array. + nested = { + "request_id": "req-1", + "events": [{"step": i, "kind": "trace", "msg": f"e{i}"} for i in range(20)], + } + out.append(("nested_object_with_array", json.dumps(nested), "", 1.0)) + + # 9. Bias > 1 (keep more) on the 30-dict case. + out.append(("dict_array_30_bias_high", json.dumps(items_30_dict), "", 1.5)) + + # 10. Bias < 1 (keep fewer) on the 30-dict case. + out.append(("dict_array_30_bias_low", json.dumps(items_30_dict), "", 0.7)) + + # 11. Unicode payload — exercises the `ensure_ascii=False` path in + # Python's safe_json_dumps. Rust's python_safe_json_dumps must emit + # raw UTF-8 bytes here, not `\uXXXX` escapes. + unicode_items = [{"id": i, "msg": f"hello 中文 русский {i}", "tag": "тест"} for i in range(20)] + out.append(("unicode_dict_array", json.dumps(unicode_items), "", 1.0)) + + # 12. Larger dict array (100 items) with a strong sequential `id` + # field — exercises top_n strategy via field stats. + big_seq = [ + {"id": i, "level": "info" if i % 7 != 0 else "warn", "message": f"seq {i}"} + for i in range(100) + ] + out.append(("dict_array_100_sequential", json.dumps(big_seq), "", 1.0)) + + # 13. Time-series-like payload: monotonic timestamp + float metric. + ts = [{"ts": 1000 + i, "metric": float(i * 1.5), "host": f"host-{i % 3}"} for i in range(50)] + out.append(("time_series_50", json.dumps(ts), "", 1.0)) + + # 14. Many duplicate items — exercises dedup_identical_items. + dups = [{"event": "heartbeat", "ok": True} for _ in range(40)] + out.append(("duplicate_dicts_40", json.dumps(dups), "", 1.0)) + + # 15. Empty array — boundary case, must round-trip cleanly. + out.append(("empty_array", json.dumps([]), "", 1.0)) + + # 16. Array of nulls and bools — non-crushable mixed type. + out.append( + ( + "nulls_and_bools", + json.dumps([None, True, False, None, True, False, None]), + "", + 1.0, + ) + ) + + # 17. Deeply nested structure: 3-level depth with arrays at each + # level. Exercises process_value's recursion. + deep = {"a": {"b": {"events": [{"i": i, "kind": "deep", "v": f"x{i}"} for i in range(15)]}}} + out.append(("nested_3deep_with_array", json.dumps(deep), "", 1.0)) + + return out + + +def main() -> int: + written: list[Path] = [] + for label, content, query, bias in _scenarios(): + path = _record(label, content, query, bias) + written.append(path) + print(f" + {path.relative_to(_REPO_ROOT)}") + print(f"wrote {len(written)} fixture(s) → {_FIXTURES_DIR.relative_to(_REPO_ROOT)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())