From e530de5ad22100bcfaa12a463961dcb08d9671c8 Mon Sep 17 00:00:00 2001 From: "Ruben A." <73822602+RubenAAA@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:21:57 +0400 Subject: [PATCH] feat(rust): port CodeCompressor AST compressor to Rust (parity-only) (#1154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds crates/headroom-core/src/transforms/code_compressor.rs (1,882 lines): the AST-aware CodeCompressor ported to Rust on tree-sitter, with grammars for Python, JavaScript, TypeScript, Go, Rust, Java, C and C++. Parity-only, like #1153. Nothing calls it: the only references outside the module are the pub mod / pub use declarations in transforms/mod.rs, and live_zone.rs still routes SourceCode to a no-op. The pyo3 bridge is untouched and no Python source changes, so the engine is unreachable from the shipped package. #1155 wires it into live-zone dispatch. Every grammar is pinned with '=' to the exact version of the corresponding Python tree-sitter- PyPI wheel. Same version on crates.io and PyPI means the same grammar.js, hence the same generated parser.c, hence node-for-node identical ASTs — the precondition for byte-parity. A canary over 9 samples x 8 languages confirmed identical node-type and line-span trees at these pins; bumping any pin requires re-running it and re-recording the fixtures. Ships 30 recorded parity fixtures, a CodeCompressorComparator in headroom-parity, and scripts/record_code_compressor_fixtures.py. Verified byte-identical to the recorded Python output: [code_aware_compressor] total=30 matched=30 skipped=0 diffed=0 Full harness on the merge result: 227 fixtures, 182 matched, 45 skipped (cache_aligner + ccr stubs), 0 diffed, exit 0 — with kompress at 21/21 under ONNX Runtime 1.24.4 (see #2591). Also verified cargo check -p headroom-core --no-default-features passes, so the static-musl path stays intact. --- .gitignore | 1 + Cargo.lock | 115 + crates/headroom-core/Cargo.toml | 19 + .../src/transforms/code_compressor.rs | 1882 +++++++++++++++++ crates/headroom-core/src/transforms/mod.rs | 5 + .../tests/code_compressor_parity.rs | 150 ++ crates/headroom-parity/src/lib.rs | 119 ++ scripts/record_code_compressor_fixtures.py | 83 + .../07dd1a61089c1e53.json | 41 + .../095be1adad76b588.json | 37 + .../1e319c5e33af4af1.json | 37 + .../1e496ebaf89fab96.json | 42 + .../2c239d4af41282c2.json | 41 + .../2d214f401618d639.json | 41 + .../36e5ff5093790e6f.json | 37 + .../3e706cfb03aa3c8e.json | 37 + .../411d5549bcd5f50d.json | 37 + .../420dd19a087526ca.json | 39 + .../53030c7c11ce0082.json | 42 + .../5c057f7bfd56af05.json | 41 + .../6c04d398f9c3014e.json | 37 + .../716a5a2380907e71.json | 37 + .../791805bff164ee9d.json | 40 + .../7cee85d20b3604dd.json | 42 + .../88fe6b4b0a9908c5.json | 37 + .../921876ad5dd26cf6.json | 40 + .../b16202e9b81a5bba.json | 37 + .../b753b4725da91f71.json | 37 + .../b8fa00d165cd059a.json | 42 + .../ba9c0fda1beda6ec.json | 39 + .../bd14bdd440e5a1ab.json | 37 + .../bfcb158739b34b43.json | 37 + .../d72fde8ea01cb782.json | 40 + .../da21f7240cb20e8b.json | 37 + .../e79625ba83a6f22d.json | 40 + .../e8ce85f0703b083b.json | 42 + .../f8a7f62203b3d8ba.json | 42 + .../f9a7c264592ff8c4.json | 37 + tests/parity/recorder.py | 445 ++++ 39 files changed, 3991 insertions(+) create mode 100644 crates/headroom-core/src/transforms/code_compressor.rs create mode 100644 crates/headroom-core/tests/code_compressor_parity.rs create mode 100644 scripts/record_code_compressor_fixtures.py create mode 100644 tests/parity/fixtures/code_aware_compressor/07dd1a61089c1e53.json create mode 100644 tests/parity/fixtures/code_aware_compressor/095be1adad76b588.json create mode 100644 tests/parity/fixtures/code_aware_compressor/1e319c5e33af4af1.json create mode 100644 tests/parity/fixtures/code_aware_compressor/1e496ebaf89fab96.json create mode 100644 tests/parity/fixtures/code_aware_compressor/2c239d4af41282c2.json create mode 100644 tests/parity/fixtures/code_aware_compressor/2d214f401618d639.json create mode 100644 tests/parity/fixtures/code_aware_compressor/36e5ff5093790e6f.json create mode 100644 tests/parity/fixtures/code_aware_compressor/3e706cfb03aa3c8e.json create mode 100644 tests/parity/fixtures/code_aware_compressor/411d5549bcd5f50d.json create mode 100644 tests/parity/fixtures/code_aware_compressor/420dd19a087526ca.json create mode 100644 tests/parity/fixtures/code_aware_compressor/53030c7c11ce0082.json create mode 100644 tests/parity/fixtures/code_aware_compressor/5c057f7bfd56af05.json create mode 100644 tests/parity/fixtures/code_aware_compressor/6c04d398f9c3014e.json create mode 100644 tests/parity/fixtures/code_aware_compressor/716a5a2380907e71.json create mode 100644 tests/parity/fixtures/code_aware_compressor/791805bff164ee9d.json create mode 100644 tests/parity/fixtures/code_aware_compressor/7cee85d20b3604dd.json create mode 100644 tests/parity/fixtures/code_aware_compressor/88fe6b4b0a9908c5.json create mode 100644 tests/parity/fixtures/code_aware_compressor/921876ad5dd26cf6.json create mode 100644 tests/parity/fixtures/code_aware_compressor/b16202e9b81a5bba.json create mode 100644 tests/parity/fixtures/code_aware_compressor/b753b4725da91f71.json create mode 100644 tests/parity/fixtures/code_aware_compressor/b8fa00d165cd059a.json create mode 100644 tests/parity/fixtures/code_aware_compressor/ba9c0fda1beda6ec.json create mode 100644 tests/parity/fixtures/code_aware_compressor/bd14bdd440e5a1ab.json create mode 100644 tests/parity/fixtures/code_aware_compressor/bfcb158739b34b43.json create mode 100644 tests/parity/fixtures/code_aware_compressor/d72fde8ea01cb782.json create mode 100644 tests/parity/fixtures/code_aware_compressor/da21f7240cb20e8b.json create mode 100644 tests/parity/fixtures/code_aware_compressor/e79625ba83a6f22d.json create mode 100644 tests/parity/fixtures/code_aware_compressor/e8ce85f0703b083b.json create mode 100644 tests/parity/fixtures/code_aware_compressor/f8a7f62203b3d8ba.json create mode 100644 tests/parity/fixtures/code_aware_compressor/f9a7c264592ff8c4.json diff --git a/.gitignore b/.gitignore index 9e5d9a79d..4aae0af7a 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ scripts/* !scripts/replay_codex_ws_load.py !scripts/export_kompress_v2_onnx.py !scripts/record_kompress_fixtures.py +!scripts/record_code_compressor_fixtures.py # Rust / Cargo build artifacts /target/ diff --git a/Cargo.lock b/Cargo.lock index 9bfc54441..5959d3596 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1899,6 +1899,15 @@ dependencies = [ "tokenizers", "toml", "tracing", + "tree-sitter", + "tree-sitter-c", + "tree-sitter-cpp", + "tree-sitter-go", + "tree-sitter-java", + "tree-sitter-javascript", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-typescript", "unidiff", ] @@ -4108,6 +4117,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strsim" version = "0.11.1" @@ -4614,6 +4629,106 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tree-sitter" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5168a515fe492af54c5cc8800ff8c840be09fa5168de45838afaecd3e008bce4" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-c" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2eb57a55fed6b00812912e730b7a275cf4fe98bfd6a5d76263d4438371728" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-java" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml index 6c8cf8691..c47d2a88b 100644 --- a/crates/headroom-core/Cargo.toml +++ b/crates/headroom-core/Cargo.toml @@ -137,6 +137,25 @@ redis = { version = "0.27", optional = true, default-features = false } # classifier live with the other Phase B/F policy primitives without # cycling through the proxy crate. Tiny crate (no I/O, just types). http = "1" +# tree-sitter + per-language grammars for the CodeCompressor AST port. +# Versions are pinned to EXACTLY match the Python reference grammars +# (`tree-sitter-` PyPI wheels) so the Rust and Python parsers emit +# node-for-node identical ASTs — the precondition for byte-parity. Same +# version number on crates.io + PyPI means the same `grammar.js` source, +# hence the same generated `parser.c`. The grammar-parity canary (9 +# samples × 8 languages) confirmed 100% identical node-type + line-span +# trees at these exact pins. Bumping any pin requires re-running the +# canary and re-recording the code_aware_compressor fixtures. +tree-sitter = "=0.25.2" +tree-sitter-python = "=0.25.0" +tree-sitter-javascript = "=0.25.0" +tree-sitter-typescript = "=0.23.2" +tree-sitter-go = "=0.25.0" +tree-sitter-rust = "=0.24.2" +tree-sitter-java = "=0.23.5" +tree-sitter-c = "=0.24.2" +tree-sitter-cpp = "=0.23.4" + # Load ONNX Runtime dynamically on every platform. The alternative, # `ort-download-binaries-*`, statically links Microsoft's prebuilt ORT: # on Windows it emits DirectML link libs (`DXCORE`, `DXGI`, `D3D12`, diff --git a/crates/headroom-core/src/transforms/code_compressor.rs b/crates/headroom-core/src/transforms/code_compressor.rs new file mode 100644 index 000000000..c06447b52 --- /dev/null +++ b/crates/headroom-core/src/transforms/code_compressor.rs @@ -0,0 +1,1882 @@ +//! CodeCompressor — Rust port of `headroom.transforms.code_compressor`. +//! +//! AST-preserving compression for source code. Unlike the ML Kompress +//! engine or the deterministic structural compressors (Log/Search/Diff), +//! this parses code into a tree-sitter AST and selectively compresses +//! function bodies while preserving imports, signatures, type definitions, +//! decorators, and top-level code. The output is guaranteed to be +//! syntactically valid (re-parsed; on ERROR/MISSING it returns the +//! original). +//! +//! # Grammar-version parity (the make-or-break invariant) +//! +//! The Python reference parses with `tree_sitter_language_pack`'s bundled +//! grammars; this port uses the per-language `tree-sitter-` crates. +//! Byte-parity requires node-for-node identical ASTs (same node `kind` +//! strings, same tree shape, same `start_point`/`end_point` rows). The +//! Cargo pins (`tree-sitter-python = "=0.25.0"`, …) match the exact PyPI +//! wheel versions the fixtures were recorded against; a canary over 9 +//! samples × 8 languages confirmed 100% identical node-type + line-span +//! trees. See `crates/headroom-core/Cargo.toml` for the full pin table. +//! +//! # Parity scope +//! +//! All slicing that the Python reference does by *byte offset into a `str`* +//! (`code[node.start_byte:node.end_byte]`) is reproduced here as correct +//! UTF-8 byte slicing (`&code[start..end]`). For ASCII inputs these are +//! identical; for non-ASCII identifiers/strings the Python path is latently +//! buggy (slices a `str` by byte index) and the two diverge. Parity +//! fixtures are therefore ASCII; non-ASCII is out of parity scope, exactly +//! as the line-based body slicing (which both sides do by row, and which is +//! always correct) sidesteps the issue for the main compression path. +//! +//! # CCR +//! +//! Like the other engines, the Rust port returns the compressed string +//! only. The Python inline `# [N tokens compressed... hash=]` marker is +//! intentionally not reproduced; live-zone CCR uses the `<>` +//! convention via the dispatcher. Parity fixtures are recorded with +//! `enable_ccr=False` so the output is deterministic and store-independent. + +use std::collections::{BTreeSet, HashMap}; + +use tree_sitter::{Language, Node, Parser, Tree}; + +// ─── Enums ────────────────────────────────────────────────────────────── + +/// Supported programming languages. `value()` matches the Python +/// `CodeLanguage` enum `.value` strings (used in the serialized result). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CodeLanguage { + Python, + Javascript, + Typescript, + Go, + Rust, + Java, + C, + Cpp, + Unknown, +} + +impl CodeLanguage { + pub fn value(self) -> &'static str { + match self { + CodeLanguage::Python => "python", + CodeLanguage::Javascript => "javascript", + CodeLanguage::Typescript => "typescript", + CodeLanguage::Go => "go", + CodeLanguage::Rust => "rust", + CodeLanguage::Java => "java", + CodeLanguage::C => "c", + CodeLanguage::Cpp => "cpp", + CodeLanguage::Unknown => "unknown", + } + } + + /// Parse from a lowercase language name. `None` for unrecognized + /// (Python raises `ValueError`; callers that need that semantics check + /// for `None`). + pub fn from_name(s: &str) -> Option { + Some(match s { + "python" => CodeLanguage::Python, + "javascript" => CodeLanguage::Javascript, + "typescript" => CodeLanguage::Typescript, + "go" => CodeLanguage::Go, + "rust" => CodeLanguage::Rust, + "java" => CodeLanguage::Java, + "c" => CodeLanguage::C, + "cpp" => CodeLanguage::Cpp, + "unknown" => CodeLanguage::Unknown, + _ => return None, + }) + } + + /// The tree-sitter grammar for this language, or `None` for `Unknown`. + fn grammar(self) -> Option { + Some(match self { + CodeLanguage::Python => tree_sitter_python::LANGUAGE.into(), + CodeLanguage::Javascript => tree_sitter_javascript::LANGUAGE.into(), + CodeLanguage::Typescript => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + CodeLanguage::Go => tree_sitter_go::LANGUAGE.into(), + CodeLanguage::Rust => tree_sitter_rust::LANGUAGE.into(), + CodeLanguage::Java => tree_sitter_java::LANGUAGE.into(), + CodeLanguage::C => tree_sitter_c::LANGUAGE.into(), + CodeLanguage::Cpp => tree_sitter_cpp::LANGUAGE.into(), + CodeLanguage::Unknown => return None, + }) + } +} + +/// How to handle Python docstrings. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DocstringMode { + Full, + FirstLine, + Remove, + /// Alias for `Remove` (deprecated). + None, +} + +impl DocstringMode { + pub fn value(self) -> &'static str { + match self { + DocstringMode::Full => "full", + DocstringMode::FirstLine => "first_line", + DocstringMode::Remove => "remove", + DocstringMode::None => "none", + } + } + + pub fn from_value(s: &str) -> Option { + Some(match s { + "full" => DocstringMode::Full, + "first_line" => DocstringMode::FirstLine, + "remove" => DocstringMode::Remove, + "none" => DocstringMode::None, + _ => return None, + }) + } +} + +// ─── Language config ──────────────────────────────────────────────────── + +/// Data-driven AST node-type tables + syntactic conventions for one +/// language. Mirrors the Python `LangConfig` frozen dataclass. +struct LangConfig { + import_nodes: &'static [&'static str], + function_nodes: &'static [&'static str], + class_nodes: &'static [&'static str], + type_nodes: &'static [&'static str], + body_node_types: &'static [&'static str], + decorator_node: Option<&'static str>, + comment_prefix: &'static str, + uses_colon_after_signature: bool, + package_node: Option<&'static str>, +} + +impl LangConfig { + fn is_import(&self, k: &str) -> bool { + self.import_nodes.contains(&k) + } + fn is_function(&self, k: &str) -> bool { + self.function_nodes.contains(&k) + } + fn is_class(&self, k: &str) -> bool { + self.class_nodes.contains(&k) + } + fn is_type(&self, k: &str) -> bool { + self.type_nodes.contains(&k) + } + fn is_body(&self, k: &str) -> bool { + self.body_node_types.contains(&k) + } +} + +/// Returns the `LangConfig` for a language, or `None` for `Unknown` +/// (mirrors `_LANG_CONFIGS.get(language)`). +fn lang_config(language: CodeLanguage) -> Option { + Some(match language { + CodeLanguage::Python => LangConfig { + import_nodes: &["import_statement", "import_from_statement"], + function_nodes: &["function_definition"], + class_nodes: &["class_definition"], + type_nodes: &["type_alias_statement"], + body_node_types: &["block"], + decorator_node: Some("decorated_definition"), + comment_prefix: "#", + uses_colon_after_signature: true, + package_node: None, + }, + CodeLanguage::Javascript => LangConfig { + import_nodes: &["import_statement", "import_declaration"], + function_nodes: &["function_declaration", "method_definition"], + class_nodes: &["class_declaration"], + type_nodes: &[], + body_node_types: &["statement_block"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: None, + }, + CodeLanguage::Typescript => LangConfig { + import_nodes: &["import_statement", "import_declaration"], + function_nodes: &["function_declaration", "method_definition"], + class_nodes: &["class_declaration"], + type_nodes: &["interface_declaration", "type_alias_declaration"], + body_node_types: &["statement_block"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: None, + }, + CodeLanguage::Go => LangConfig { + import_nodes: &["import_declaration"], + function_nodes: &["function_declaration", "method_declaration"], + class_nodes: &[], + type_nodes: &["type_declaration"], + body_node_types: &["block"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: Some("package_clause"), + }, + CodeLanguage::Rust => LangConfig { + import_nodes: &["use_declaration"], + function_nodes: &["function_item"], + class_nodes: &["impl_item"], + type_nodes: &["struct_item", "enum_item", "type_item", "trait_item"], + body_node_types: &["block"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: None, + }, + CodeLanguage::Java => LangConfig { + import_nodes: &["import_declaration"], + function_nodes: &["method_declaration", "constructor_declaration"], + class_nodes: &["class_declaration", "interface_declaration"], + type_nodes: &["enum_declaration"], + body_node_types: &["block"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: Some("package_declaration"), + }, + CodeLanguage::C => LangConfig { + import_nodes: &["preproc_include"], + function_nodes: &["function_definition"], + class_nodes: &[], + type_nodes: &["struct_specifier", "enum_specifier", "type_definition"], + body_node_types: &["compound_statement"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: None, + }, + CodeLanguage::Cpp => LangConfig { + import_nodes: &["preproc_include"], + function_nodes: &["function_definition"], + class_nodes: &["class_specifier"], + type_nodes: &["struct_specifier", "enum_specifier", "type_definition"], + body_node_types: &["compound_statement"], + decorator_node: None, + comment_prefix: "//", + uses_colon_after_signature: false, + package_node: None, + }, + CodeLanguage::Unknown => return None, + }) +} + +// ─── Config ───────────────────────────────────────────────────────────── + +/// Configuration for code-aware compression. Field defaults match the +/// Python `CodeCompressorConfig` dataclass. The `preserve_*` and +/// `compress_comments` flags are vestigial in the reference (the extractor +/// always preserves structure regardless), kept here for fidelity. +#[derive(Debug, Clone)] +pub struct CodeCompressorConfig { + pub preserve_imports: bool, + pub preserve_signatures: bool, + pub preserve_type_annotations: bool, + pub preserve_decorators: bool, + pub docstring_mode: DocstringMode, + pub target_compression_rate: f64, + pub max_body_lines: i64, + pub compress_comments: bool, + pub min_tokens_for_compression: i64, + pub language_hint: Option, + pub fallback_to_kompress: bool, + pub semantic_analysis: bool, + pub enable_ccr: bool, + pub ccr_ttl: i64, +} + +impl Default for CodeCompressorConfig { + fn default() -> Self { + Self { + preserve_imports: true, + preserve_signatures: true, + preserve_type_annotations: true, + preserve_decorators: true, + docstring_mode: DocstringMode::FirstLine, + target_compression_rate: 0.2, + max_body_lines: 5, + compress_comments: true, + min_tokens_for_compression: 100, + language_hint: None, + fallback_to_kompress: true, + semantic_analysis: true, + enable_ccr: true, + ccr_ttl: 300, + } + } +} + +// ─── Result ───────────────────────────────────────────────────────────── + +/// Result of code-aware compression. Field set + serialization mirror the +/// Python `CodeCompressionResult` dataclass (via `asdict`); the `@property` +/// derivatives (`tokens_saved`, `savings_percentage`, `summary`) are not +/// serialized and live as methods here. +#[derive(Debug, Clone, PartialEq)] +pub struct CodeCompressionResult { + pub compressed: String, + pub original: String, + pub original_tokens: i64, + pub compressed_tokens: i64, + pub compression_ratio: f64, + pub language: CodeLanguage, + pub language_confidence: f64, + pub preserved_imports: i64, + pub preserved_signatures: i64, + pub compressed_bodies: i64, + pub syntax_valid: bool, + pub cache_key: Option, + /// Short-name → score (round-3), insertion order is irrelevant to the + /// serialized object (compared as a map). + pub symbol_scores: Vec<(String, f64)>, +} + +impl CodeCompressionResult { + pub fn tokens_saved(&self) -> i64 { + (self.original_tokens - self.compressed_tokens).max(0) + } + + pub fn savings_percentage(&self) -> f64 { + if self.original_tokens == 0 { + 0.0 + } else { + (self.tokens_saved() as f64 / self.original_tokens as f64) * 100.0 + } + } +} + +// ─── Extracted structure ──────────────────────────────────────────────── + +/// `(compressed, structure, symbol_scores)` — the result of an AST pass. +type AstCompression = (String, CodeStructure, Vec<(String, f64)>); + +#[derive(Default)] +struct CodeStructure { + imports: Vec, + type_definitions: Vec, + class_definitions: Vec, + function_signatures: Vec, + /// (signature, body, line) — never populated by the current paths, so + /// `compressed_bodies` is always 0 (mirrors the reference). + function_bodies: Vec<(String, String, i64)>, + top_level_code: Vec, + other: Vec, +} + +// ─── Symbol analysis ──────────────────────────────────────────────────── + +#[derive(Default)] +struct SymbolAnalysis { + /// qname → normalized score (round-3), insertion order preserved. + scores: Vec<(String, f64)>, + /// qname → set of short names it calls, insertion order preserved + /// (matters for `make_omitted_comment` first-match selection). + calls: Vec<(String, BTreeSet)>, + /// qname → short name. + bare_names: HashMap, + /// qname → body line count. + body_line_counts: HashMap, +} + +impl SymbolAnalysis { + fn calls_of(&self, qname: &str) -> Option<&BTreeSet> { + self.calls.iter().find(|(k, _)| k == qname).map(|(_, v)| v) + } +} + +// ─── Module helpers (stateless) ───────────────────────────────────────── + +/// `code[node.start_byte:node.end_byte]` — correct UTF-8 byte slice. +fn node_text<'a>(node: Node, code: &'a str) -> &'a str { + &code[node.start_byte()..node.end_byte()] +} + +/// First child whose kind is a name token; returns its (real) text. Mirrors +/// `_get_definition_name`. +fn get_definition_name(node: Node, code: &str) -> Option { + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + let k = child.kind(); + if k == "identifier" || k == "name" || k == "type_identifier" || k == "property_identifier" + { + return Some(node_text(child, code).to_string()); + } + } + None +} + +fn is_public_symbol(name: &str, language: CodeLanguage) -> bool { + if name.is_empty() { + return false; + } + if language == CodeLanguage::Go { + return name.chars().next().is_some_and(|c| c.is_uppercase()); + } + !name.starts_with('_') +} + +/// Look up the allocated body-line limit for a function. `max_body_lines` +/// always acts as a hard cap. Mirrors `_get_body_limit`. +fn get_body_limit( + func_name: Option<&str>, + body_limits: &HashMap, + max_body_lines: i64, +) -> i64 { + if let Some(name) = func_name { + if !body_limits.is_empty() { + if let Some(&v) = body_limits.get(name) { + return v.min(max_body_lines); + } + } + } + max_body_lines +} + +/// Detect the indentation used in a list of code lines. Mirrors `_detect_indent`. +fn detect_indent(lines: &[&str]) -> String { + for line in lines { + if !line.trim().is_empty() { + return leading_ws(line).to_string(); + } + } + " ".to_string() +} + +/// Leading-whitespace prefix of a line (`line[:len-len(lstrip)]`). +fn leading_ws(line: &str) -> &str { + &line[..line.len() - line.trim_start().len()] +} + +/// Build the omitted-body comment with call info. Mirrors `_make_omitted_comment`. +fn make_omitted_comment( + func_name: Option<&str>, + omitted_count: i64, + indent: &str, + comment_prefix: &str, + analysis: &SymbolAnalysis, +) -> String { + let mut calls_info = String::new(); + if let Some(func_name) = func_name { + // Candidate keys: the bare name first, then every qname ending in + // `.func_name` (insertion order). Pick the first present in `calls`. + let suffix = format!(".{func_name}"); + let mut candidates: Vec<&str> = vec![func_name]; + for (k, _) in &analysis.calls { + if k.ends_with(&suffix) { + candidates.push(k.as_str()); + } + } + for key in candidates { + if let Some(called) = analysis.calls_of(key) { + if !called.is_empty() { + // BTreeSet iterates sorted == Python sorted(called). + let sorted_calls: Vec<&str> = + called.iter().take(5).map(|s| s.as_str()).collect(); + calls_info = format!("; calls: {}", sorted_calls.join(", ")); + if called.len() > 5 { + calls_info.push_str(&format!(" +{} more", called.len() - 5)); + } + } + break; + } + } + } + format!("{indent}{comment_prefix} [{omitted_count} lines omitted{calls_info}]") +} + +/// Count ERROR + MISSING nodes (recursive). Mirrors `_count_error_nodes`. +fn count_error_nodes(node: Node) -> i64 { + let mut count = 0; + if node.kind() == "ERROR" || node.is_missing() { + count += 1; + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + count += count_error_nodes(child); + } + count +} + +/// True if the tree contains an ERROR or MISSING node. Mirrors `_has_syntax_issues`. +fn has_syntax_issues(node: Node) -> bool { + if node.kind() == "ERROR" || node.is_missing() { + return true; + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if has_syntax_issues(child) { + return true; + } + } + false +} + +/// CPython `round(x)` (ndigits=None): nearest int, ties to even. +fn py_round_int(x: f64) -> i64 { + let r = x.round(); // half away from zero == C round() + if (x - r).abs() == 0.5 { + // Halfway case: round to even. + (2.0 * (x / 2.0).round()) as i64 + } else { + r as i64 + } +} + +/// CPython `round(x, 3)`: correctly-rounded to 3 decimals, ties to even. +/// Rust's `{:.3}` formatter is correctly rounded half-to-even, so format + +/// re-parse yields the same f64 CPython's dtoa-based rounding produces. +fn py_round3(x: f64) -> f64 { + format!("{x:.3}").parse::().unwrap() +} + +/// `len(text) // 4` over Unicode code points, min 1. Mirrors +/// `_estimate_tokens` with `tokenizer=None`. +fn estimate_tokens(text: &str) -> i64 { + ((text.chars().count() / 4).max(1)) as i64 +} + +/// Parse `code` with the grammar for `language`. `None` for `Unknown` or on +/// parse failure. +fn parse_code(code: &str, language: CodeLanguage) -> Option { + let grammar = language.grammar()?; + let mut parser = Parser::new(); + parser.set_language(&grammar).ok()?; + parser.parse(code.as_bytes(), None) +} + +/// First `n` Unicode code points of `s` (Python `s[:n]` for a `str`). +fn char_prefix(s: &str, n: usize) -> String { + s.chars().take(n).collect() +} + +// ─── Language detection ───────────────────────────────────────────────── + +mod prefilter { + use std::sync::OnceLock; + + use regex::Regex; + + use super::CodeLanguage; + + /// (language, patterns) in the Python `_LANGUAGE_PREFILTER` insertion + /// order — that order is the stable tie-break for detection. + pub fn patterns() -> &'static [(CodeLanguage, Vec)] { + static P: OnceLock)>> = OnceLock::new(); + P.get_or_init(|| { + let c = |s: &str| Regex::new(s).unwrap(); + vec![ + ( + CodeLanguage::Python, + vec![ + c(r"(?m)^\s*(def|class|import|from|async def)\s+\w+"), + c(r"(?m)^\s*@\w+"), + c(r#"(?m)^\s*""""#), + c(r"(?m)^\s*if __name__\s*=="), + ], + ), + ( + CodeLanguage::Javascript, + vec![ + c(r"(?m)^\s*(function|const|let|var|class|export)\s+\w+"), + c(r"(?m)^\s*async\s+(function|=>)"), + c(r"(?m)^\s*module\.exports"), + c(r#"(?m)^\s*(import|export)\s+.*\s+from\s+['"]"#), + ], + ), + ( + CodeLanguage::Typescript, + vec![ + c(r"(?m)^\s*(interface|type|enum|namespace)\s+\w+"), + c(r"(?m):\s*(string|number|boolean|any|void|Promise)\b"), + ], + ), + ( + CodeLanguage::Go, + vec![ + c(r"(?m)^\s*(func|type|package|import)\s+"), + c(r"(?m)^\s*func\s+\([^)]+\)\s+\w+"), + c(r"(?m)\bstruct\s*\{"), + ], + ), + ( + CodeLanguage::Rust, + vec![ + c(r"(?m)^\s*(fn|struct|enum|impl|mod|use|pub)\s+"), + c(r"(?m)^\s*#\["), + ], + ), + ( + CodeLanguage::Java, + vec![ + c(r"(?m)^\s*(public|private|protected)\s+(class|interface|enum)"), + c(r"(?m)^\s*package\s+[\w.]+;"), + ], + ), + ( + CodeLanguage::C, + vec![ + c(r#"(?m)^\s*#include\s*[<"]"#), + c(r"(?m)^\s*(int|void|char|float|double)\s+\w+\s*\("), + c(r"(?m)^\s*typedef\s+"), + ], + ), + ( + CodeLanguage::Cpp, + vec![ + c(r#"(?m)^\s*#include\s*[<"]"#), + c(r"(?m)\bnamespace\s+\w+"), + c(r"(?m)::\w+"), + ], + ), + ] + }) + } +} + +/// Detect the language of `code`. Mirrors `detect_language`: regex prefilter +/// → tree-sitter fewest-errors → regex-only fallback. +pub fn detect_language(code: &str) -> (CodeLanguage, f64) { + if code.trim().is_empty() { + return (CodeLanguage::Unknown, 0.0); + } + + let sample = char_prefix(code, 5000); + + // Phase 1: prefilter scores, in fixed enum order. + let mut candidates: Vec<(CodeLanguage, i64)> = Vec::new(); + for (lang, pats) in prefilter::patterns() { + let mut score = 0i64; + for pat in pats { + score += pat.find_iter(&sample).count() as i64; + } + if score > 0 { + candidates.push((*lang, score)); + } + } + + if candidates.is_empty() { + return (CodeLanguage::Unknown, 0.0); + } + + // Disambiguation: TS superset of JS; C++ superset of C. + let get = |cs: &[(CodeLanguage, i64)], l: CodeLanguage| { + cs.iter().find(|(x, _)| *x == l).map(|(_, s)| *s) + }; + if let (Some(ts), Some(_js)) = ( + get(&candidates, CodeLanguage::Typescript), + get(&candidates, CodeLanguage::Javascript), + ) { + if ts >= 2 { + if let Some(e) = candidates + .iter_mut() + .find(|(x, _)| *x == CodeLanguage::Javascript) + { + e.1 = 0; + } + } + } + if let (Some(cpp), Some(_c)) = ( + get(&candidates, CodeLanguage::Cpp), + get(&candidates, CodeLanguage::C), + ) { + if cpp >= 2 { + if let Some(e) = candidates.iter_mut().find(|(x, _)| *x == CodeLanguage::C) { + e.1 = 0; + } + } + } + + // Phase 2: tree-sitter, fewest errors then most top-level children. + // (tree-sitter is always available in the Rust port.) + let code_bytes_src = char_prefix(code, 10000); + let mut best_lang = CodeLanguage::Unknown; + let mut min_errors = i64::MAX; + let mut best_node_count: i64 = 0; + + // Sort candidates by score desc, stable (preserves enum order on ties). + let mut sorted_candidates = candidates.clone(); + sorted_candidates.sort_by_key(|&(_, s)| std::cmp::Reverse(s)); + + for (lang, _score) in &sorted_candidates { + if *lang == CodeLanguage::Unknown || get(&candidates, *lang) == Some(0) { + continue; + } + let Some(tree) = parse_code(&code_bytes_src, *lang) else { + continue; + }; + let root = tree.root_node(); + let error_count = count_error_nodes(root); + let node_count = root.child_count() as i64; + if error_count < min_errors || (error_count == min_errors && node_count > best_node_count) { + min_errors = error_count; + best_lang = *lang; + best_node_count = node_count; + } + } + + if best_lang != CodeLanguage::Unknown { + let total_lines = (code.trim().split('\n').count() as i64).max(1); + let error_ratio = min_errors as f64 / total_lines as f64; + // Python: max(0.3, min(1.0, 1.0 - error_ratio)) == clamp(0.3, 1.0). + let confidence = (1.0 - error_ratio).clamp(0.3, 1.0); + return (best_lang, confidence); + } + + // Phase 3: regex-only fallback (first max in insertion order). + let mut best = candidates[0]; + for &cand in &candidates[1..] { + if cand.1 > best.1 { + best = cand; + } + } + if best.1 == 0 { + return (CodeLanguage::Unknown, 0.0); + } + let confidence = (0.3 + best.1 as f64 * 0.1).min(1.0); + (best.0, confidence) +} + +// ─── Compressor ───────────────────────────────────────────────────────── + +/// AST-preserving code compressor. Construct with a config and call +/// [`CodeAwareCompressor::compress`]. +#[derive(Debug, Clone)] +pub struct CodeAwareCompressor { + pub config: CodeCompressorConfig, +} + +/// Shared per-compression context (source + parsed tables), to keep the +/// traversal methods' signatures small. +struct Ctx<'a> { + code: &'a str, + code_lines: Vec<&'a str>, + language: CodeLanguage, + lang: &'a LangConfig, + body_limits: &'a HashMap, + analysis: &'a SymbolAnalysis, + config: &'a CodeCompressorConfig, +} + +impl CodeAwareCompressor { + pub fn new(config: CodeCompressorConfig) -> Self { + Self { config } + } + + /// Compress with all defaults (language auto-detect, no context). This + /// is the path the parity recorder exercises (`compress(code)`). + pub fn compress(&self, code: &str) -> CodeCompressionResult { + self.compress_with(code, None, "") + } + + /// Full compression entry point. `language` overrides detection; + /// `context` boosts symbol importance for matching names. Mirrors + /// `CodeAwareCompressor.compress` (with `tokenizer=None`). + pub fn compress_with( + &self, + code: &str, + language: Option<&str>, + context: &str, + ) -> CodeCompressionResult { + if code.trim().is_empty() { + return passthrough_result(code, 0, CodeLanguage::Unknown, 0.0); + } + + let original_tokens = estimate_tokens(code); + + if original_tokens < self.config.min_tokens_for_compression { + return passthrough_result(code, original_tokens, CodeLanguage::Unknown, 0.0); + } + + // Detect or use specified language. + let (detected_lang, confidence) = if let Some(lang) = language { + match CodeLanguage::from_name(&lang.to_lowercase()) { + Some(l) => (l, 1.0), + None => (CodeLanguage::Unknown, 1.0), + } + } else if let Some(hint) = &self.config.language_hint { + match CodeLanguage::from_name(&hint.to_lowercase()) { + Some(l) => (l, 1.0), + None => (CodeLanguage::Unknown, 1.0), + } + } else { + detect_language(code) + }; + + // Unknown language → fallback (Kompress) or passthrough. + if detected_lang == CodeLanguage::Unknown { + // Kompress fallback is delegated to the live-zone dispatcher in + // the Rust port (the engine doesn't own the model); fixtures are + // recorded with fallback_to_kompress=False, so this is the + // passthrough branch. + return CodeCompressionResult { + language: CodeLanguage::Unknown, + language_confidence: 0.0, + ..passthrough_result(code, original_tokens, CodeLanguage::Unknown, 0.0) + }; + } + + // Parse + compress (tree-sitter always available here). + let Some((compressed, structure, symbol_scores)) = + self.compress_with_ast(code, detected_lang, context) + else { + // AST exception path → fallback/passthrough. + return passthrough_result(code, original_tokens, detected_lang, confidence); + }; + + let compressed_tokens = estimate_tokens(&compressed); + + // Verify syntax validity (ERROR + MISSING). + let syntax_valid = self.verify_syntax(&compressed, detected_lang); + if !syntax_valid { + return passthrough_result(code, original_tokens, detected_lang, confidence); + } + + let ratio = compressed_tokens as f64 / original_tokens.max(1) as f64; + + // Guard against over-aggressive compression (data loss). + if ratio < 0.05 { + return passthrough_result(code, original_tokens, detected_lang, confidence); + } + + // CCR offload (enable_ccr && ratio < 0.8) is owned by the dispatcher + // in the Rust port; fixtures record with enable_ccr=False so + // `cache_key` stays None and no marker is appended. + + CodeCompressionResult { + compressed, + original: code.to_string(), + original_tokens, + compressed_tokens, + compression_ratio: ratio, + language: detected_lang, + language_confidence: confidence, + preserved_imports: structure.imports.len() as i64, + preserved_signatures: structure.function_signatures.len() as i64, + compressed_bodies: structure.function_bodies.len() as i64, + syntax_valid, + cache_key: None, + symbol_scores, + } + } + + /// Parse + analyze + extract + assemble. Returns + /// `(compressed, structure, symbol_scores)`. `None` mirrors the Python + /// `except Exception` fallback (here only reachable on a parse miss). + fn compress_with_ast( + &self, + code: &str, + language: CodeLanguage, + context: &str, + ) -> Option { + let tree = parse_code(code, language)?; + let root = tree.root_node(); + + let analysis = self.analyze_symbol_importance(root, code, language, context); + let body_limits = self.allocate_body_budget(&analysis, code); + + let lang = lang_config(language); + let (structure, symbol_scores) = if let Some(lang) = lang { + let code_lines: Vec<&str> = code.split('\n').collect(); + let ctx = Ctx { + code, + code_lines, + language, + lang: &lang, + body_limits: &body_limits, + analysis: &analysis, + config: &self.config, + }; + let structure = ctx.extract_structure(root); + // Expose scores under short names (max per short name). + let mut symbol_scores: Vec<(String, f64)> = Vec::new(); + for (qname, score) in &analysis.scores { + let short = analysis + .bare_names + .get(qname) + .cloned() + .unwrap_or_else(|| qname.clone()); + if let Some(existing) = symbol_scores.iter_mut().find(|(k, _)| *k == short) { + if *score > existing.1 { + existing.1 = *score; + } + } else { + symbol_scores.push((short, *score)); + } + } + (structure, symbol_scores) + } else { + (extract_generic_structure(code), Vec::new()) + }; + + let compressed = assemble_compressed(&structure); + Some((compressed, structure, symbol_scores)) + } + + /// Verify that `code` re-parses without ERROR/MISSING. Mirrors `_verify_syntax`. + fn verify_syntax(&self, code: &str, language: CodeLanguage) -> bool { + match parse_code(code, language) { + Some(tree) => !has_syntax_issues(tree.root_node()), + None => false, + } + } +} + +/// Build a passthrough result (compressed == original). Used for every +/// short-circuit / fallback branch. +fn passthrough_result( + code: &str, + original_tokens: i64, + language: CodeLanguage, + confidence: f64, +) -> CodeCompressionResult { + CodeCompressionResult { + compressed: code.to_string(), + original: code.to_string(), + original_tokens, + compressed_tokens: original_tokens, + compression_ratio: 1.0, + language, + language_confidence: confidence, + preserved_imports: 0, + preserved_signatures: 0, + compressed_bodies: 0, + syntax_valid: true, + cache_key: None, + symbol_scores: Vec::new(), + } +} + +fn extract_generic_structure(code: &str) -> CodeStructure { + CodeStructure { + other: code.split('\n').map(|s| s.to_string()).collect(), + ..Default::default() + } +} + +/// Assemble compressed code from structure. Mirrors `_assemble_compressed`. +fn assemble_compressed(structure: &CodeStructure) -> String { + let mut parts: Vec = Vec::new(); + let push_section = |parts: &mut Vec, section: &[String]| { + if !section.is_empty() { + parts.extend(section.iter().cloned()); + parts.push(String::new()); + } + }; + push_section(&mut parts, &structure.imports); + push_section(&mut parts, &structure.type_definitions); + push_section(&mut parts, &structure.class_definitions); + push_section(&mut parts, &structure.function_signatures); + push_section(&mut parts, &structure.top_level_code); + if !structure.other.is_empty() { + parts.extend(structure.other.iter().cloned()); + } + // Remove trailing blank lines. + while let Some(last) = parts.last() { + if last.trim().is_empty() { + parts.pop(); + } else { + break; + } + } + parts.join("\n") +} + +// ─── Symbol importance + body budget (impl CodeAwareCompressor) ───────── + +/// Insertion-ordered put: update value if key present (keeps position), +/// else append. Mirrors Python dict assignment semantics. +fn ordered_put(v: &mut Vec<(String, V)>, key: String, val: V) { + if let Some(e) = v.iter_mut().find(|(k, _)| *k == key) { + e.1 = val; + } else { + v.push((key, val)); + } +} + +impl CodeAwareCompressor { + /// Distribution-based symbol importance. Mirrors `_analyze_symbol_importance`. + fn analyze_symbol_importance( + &self, + root: Node, + code: &str, + language: CodeLanguage, + context: &str, + ) -> SymbolAnalysis { + if !self.config.semantic_analysis { + return SymbolAnalysis::default(); + } + let Some(lang) = lang_config(language) else { + return SymbolAnalysis::default(); + }; + + let is_def = |k: &str| lang.is_function(k) || lang.is_class(k); + + // Pass 1: collect definitions with qualified names (DFS, ordered). + let mut definitions: Vec<(String, Node)> = Vec::new(); + let mut bare_names: HashMap = HashMap::new(); + collect_definitions( + root, + "", + code, + &is_def, + lang.decorator_node, + &mut definitions, + &mut bare_names, + ); + if definitions.is_empty() { + return SymbolAnalysis::default(); + } + + // Pass 2: collect all identifiers (short name → count). + let mut all_identifiers: HashMap = HashMap::new(); + collect_identifiers(root, code, &mut all_identifiers); + + // Pass 3: call relationships + body sizes. + let defined_short_names: BTreeSet = bare_names.values().cloned().collect(); + let mut function_calls: Vec<(String, BTreeSet)> = Vec::new(); + let mut body_line_counts: HashMap = HashMap::new(); + for (qname, node) in &definitions { + let func_short = bare_names.get(qname).cloned().unwrap_or_default(); + let mut calls: BTreeSet = BTreeSet::new(); + collect_calls(*node, code, &defined_short_names, &func_short, &mut calls); + function_calls.push((qname.clone(), calls)); + let text = node_text(*node, code); + let line_count = text.split('\n').count() as i64; + body_line_counts.insert(qname.clone(), (line_count - 2).max(1)); + } + + // Reference counts: subtract definition occurrences. + let mut short_name_def_count: HashMap = HashMap::new(); + for short in bare_names.values() { + *short_name_def_count.entry(short.clone()).or_insert(0) += 1; + } + let mut ref_counts: HashMap = HashMap::new(); + for (qname, _) in &definitions { + let short = &bare_names[qname]; + let count = *all_identifiers.get(short).unwrap_or(&0); + let def_count = *short_name_def_count.get(short).unwrap_or(&1); + ref_counts.insert(qname.clone(), (count - def_count).max(0)); + } + + // Context words (empty when context is ""). + let context_lower = context.to_lowercase(); + let context_words: BTreeSet = if context.is_empty() { + BTreeSet::new() + } else { + static SPLIT: std::sync::OnceLock = std::sync::OnceLock::new(); + let re = SPLIT.get_or_init(|| regex::Regex::new(r#"[\s,;:.()\[\]{}"']+"#).unwrap()); + re.split(&context_lower) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect() + }; + + // Raw importance signals per symbol. + let mut raw_signals: Vec<(String, f64)> = Vec::new(); + for (qname, _) in &definitions { + let short = bare_names[qname].clone(); + let refs = *ref_counts.get(qname).unwrap_or(&0); + let fan_out = function_calls + .iter() + .find(|(k, _)| k == qname) + .map(|(_, s)| s.len()) + .unwrap_or(0) as f64; + let is_public = is_public_symbol(&short, language); + + let mut raw = refs as f64; + raw += if is_public { 1.0 } else { 0.0 }; + raw += fan_out * 0.5; + + // Language conventions are mutually exclusive, so collapsing the + // nested guards preserves the reference's branch behavior. + if language == CodeLanguage::Python && short.starts_with("__") && short.ends_with("__") + { + raw += 2.0; + } else if language == CodeLanguage::Go + && short.chars().next().is_some_and(|c| c.is_uppercase()) + { + raw += 1.0; + } + + if !context_words.is_empty() { + let name_lower = short.to_lowercase(); + if context_words.contains(&name_lower) + || (name_lower.chars().count() > 3 && context_lower.contains(&name_lower)) + { + raw += 3.0; + } + } + raw_signals.push((qname.clone(), raw)); + } + + // Min-max normalization to [0, 1], round-3. + let values: Vec = raw_signals.iter().map(|(_, v)| *v).collect(); + let min_val = values.iter().cloned().fold(f64::INFINITY, f64::min); + let max_val = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max); + let range_val = max_val - min_val; + + let mut scores: Vec<(String, f64)> = Vec::new(); + if range_val > 0.0 { + for (name, v) in &raw_signals { + scores.push((name.clone(), py_round3((v - min_val) / range_val))); + } + } else { + for (name, _) in &raw_signals { + scores.push((name.clone(), 0.5)); + } + } + + SymbolAnalysis { + scores, + calls: function_calls, + bare_names, + body_line_counts, + } + } + + /// Allocate per-symbol body-line budgets. Mirrors `_allocate_body_budget`. + fn allocate_body_budget(&self, analysis: &SymbolAnalysis, code: &str) -> HashMap { + if analysis.scores.is_empty() || analysis.body_line_counts.is_empty() { + return HashMap::new(); + } + let target_rate = self.config.target_compression_rate; + let total_lines = code.trim().split('\n').count() as i64; + let total_body_lines: i64 = analysis.body_line_counts.values().sum(); + let fixed_lines = (total_lines - total_body_lines).max(0); + let target_total = total_lines as f64 * target_rate; + let body_budget = (target_total - fixed_lines as f64).max(0.0); + + if total_body_lines == 0 { + return HashMap::new(); + } + + let score_floor = 0.05; + // weights keyed by qname, in scores order. + let mut weights: Vec<(String, f64)> = Vec::new(); + for (name, score) in &analysis.scores { + let s = score.max(score_floor); + let size = *analysis.body_line_counts.get(name).unwrap_or(&0); + weights.push((name.clone(), s * size as f64)); + } + let total_weight: f64 = weights.iter().map(|(_, w)| *w).sum(); + + let mut limits: HashMap = HashMap::new(); + if total_weight == 0.0 { + let per_func = (body_budget / (analysis.scores.len().max(1) as f64)) + .trunc() + .max(0.0) as i64; + for (name, _) in &analysis.scores { + let size = *analysis.body_line_counts.get(name).unwrap_or(&0); + limits.insert(name.clone(), per_func.min(size)); + } + return limits; + } + + for (qname, _) in &analysis.scores { + let weight = weights + .iter() + .find(|(k, _)| k == qname) + .map(|(_, w)| *w) + .unwrap_or(0.0); + let allocation = body_budget * weight / total_weight; + let max_lines = *analysis.body_line_counts.get(qname).unwrap_or(&0); + let limit = py_round_int(allocation).min(max_lines); + limits.insert(qname.clone(), limit); + let short = analysis + .bare_names + .get(qname) + .cloned() + .unwrap_or_else(|| qname.clone()); + match limits.get(&short) { + Some(&existing) if limit <= existing => {} + _ => { + limits.insert(short, limit); + } + } + } + limits + } +} + +/// DFS collect of qualified definition names → node. Mirrors the nested +/// `collect_definitions` closure. +fn collect_definitions<'t>( + node: Node<'t>, + parent_name: &str, + code: &str, + is_def: &dyn Fn(&str) -> bool, + decorator_node: Option<&str>, + definitions: &mut Vec<(String, Node<'t>)>, + bare_names: &mut HashMap, +) { + let nt = node.kind(); + if is_def(nt) { + if let Some(short) = get_definition_name(node, code) { + let qualified = if parent_name.is_empty() { + short.clone() + } else { + format!("{parent_name}.{short}") + }; + ordered_put(definitions, qualified.clone(), node); + bare_names.insert(qualified.clone(), short); + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_definitions( + child, + &qualified, + code, + is_def, + decorator_node, + definitions, + bare_names, + ); + } + return; + } + } + if let Some(dn) = decorator_node { + if nt == dn { + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if is_def(child.kind()) { + if let Some(short) = get_definition_name(child, code) { + let qualified = if parent_name.is_empty() { + short.clone() + } else { + format!("{parent_name}.{short}") + }; + ordered_put(definitions, qualified.clone(), child); + bare_names.insert(qualified.clone(), short); + let mut gc = child.walk(); + for grandchild in child.children(&mut gc) { + collect_definitions( + grandchild, + &qualified, + code, + is_def, + decorator_node, + definitions, + bare_names, + ); + } + return; + } + } + } + } + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_definitions( + child, + parent_name, + code, + is_def, + decorator_node, + definitions, + bare_names, + ); + } +} + +/// DFS count of identifier-like nodes by (real) text. Mirrors `collect_identifiers`. +fn collect_identifiers(node: Node, code: &str, out: &mut HashMap) { + let k = node.kind(); + if k == "identifier" || k == "property_identifier" || k == "type_identifier" { + let name = node_text(node, code).to_string(); + *out.entry(name).or_insert(0) += 1; + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_identifiers(child, code, out); + } +} + +/// DFS collect of calls within a function. Mirrors `collect_calls_in_function`. +fn collect_calls( + node: Node, + code: &str, + defined_short_names: &BTreeSet, + func_short: &str, + calls: &mut BTreeSet, +) { + let k = node.kind(); + if k == "identifier" || k == "property_identifier" { + let name = node_text(node, code); + if defined_short_names.contains(name) && name != func_short { + calls.insert(name.to_string()); + } + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_calls(child, code, defined_short_names, func_short, calls); + } +} + +// ─── Structure extraction (impl Ctx) ──────────────────────────────────── + +impl<'a> Ctx<'a> { + fn node_text(&self, node: Node) -> &'a str { + &self.code[node.start_byte()..node.end_byte()] + } + + /// Lines `code_lines[start..=end]` joined with `\n`. + fn lines_joined(&self, start: usize, end_inclusive: usize) -> String { + self.code_lines[start..=end_inclusive].join("\n") + } + + /// Extract structure from the AST. Mirrors `_extract_structure`. + fn extract_structure(&self, root: Node) -> CodeStructure { + let mut structure = CodeStructure::default(); + let mut captured: std::collections::HashSet<(usize, usize)> = + std::collections::HashSet::new(); + self.visit(root, &mut structure, &mut captured); + + // Top-level children not captured → top_level_code. + let mut cursor = root.walk(); + for child in root.children(&mut cursor) { + let range = (child.start_byte(), child.end_byte()); + if !captured.contains(&range) { + let text = self.node_text(child).trim(); + if !text.is_empty() { + structure.top_level_code.push(text.to_string()); + } + } + } + structure + } + + fn visit( + &self, + node: Node, + structure: &mut CodeStructure, + captured: &mut std::collections::HashSet<(usize, usize)>, + ) { + let nt = node.kind(); + let range = (node.start_byte(), node.end_byte()); + + // Package declarations (Go, Java). + if self.lang.package_node == Some(nt) { + structure + .imports + .insert(0, self.node_text(node).to_string()); + captured.insert(range); + return; + } + // Import statements. + if self.lang.is_import(nt) { + structure.imports.push(self.node_text(node).to_string()); + captured.insert(range); + return; + } + // Export statements (JS/TS). + if nt == "export_statement" { + let text = self.node_text(node).to_string(); + let mut has_func_or_class = false; + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if self.lang.is_function(child.kind()) || self.lang.is_class(child.kind()) { + has_func_or_class = true; + let compressed = self.compress_function_ast(child); + let export_prefix = &self.code[node.start_byte()..child.start_byte()]; + let export_suffix = &self.code[child.end_byte()..node.end_byte()]; + structure + .function_signatures + .push(format!("{export_prefix}{compressed}{export_suffix}")); + break; + } + } + if !has_func_or_class { + structure.imports.push(text); + } + captured.insert(range); + return; + } + // Decorated definitions (Python). + if self.lang.decorator_node == Some(nt) { + let mut decorator_text: Vec = Vec::new(); + let mut definition_compressed: Option = None; + let mut has_class_child = false; + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + let ck = child.kind(); + if ck == "decorator" { + decorator_text.push(self.node_text(child).to_string()); + } else if self.lang.is_function(ck) { + definition_compressed = Some(self.compress_function_ast(child)); + } else if self.lang.is_class(ck) { + definition_compressed = Some(self.compress_class_ast(child)); + } + if self.lang.is_class(ck) { + has_class_child = true; + } + } + match definition_compressed { + Some(def) if !decorator_text.is_empty() => { + let full_def = format!("{}\n{}", decorator_text.join("\n"), def); + if has_class_child { + structure.class_definitions.push(full_def); + } else { + structure.function_signatures.push(full_def); + } + } + Some(def) => structure.function_signatures.push(def), + None => {} + } + captured.insert(range); + return; + } + // Function/method definitions. + if self.lang.is_function(nt) { + let compressed = self.compress_function_ast(node); + structure.function_signatures.push(compressed); + captured.insert(range); + return; + } + // Class definitions. + if self.lang.is_class(nt) { + let compressed = self.compress_class_ast(node); + structure.class_definitions.push(compressed); + captured.insert(range); + return; + } + // Type definitions. + if self.lang.is_type(nt) { + structure + .type_definitions + .push(self.node_text(node).to_string()); + captured.insert(range); + return; + } + // Recurse. + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + self.visit(child, structure, captured); + } + } + + /// Compress a function/method body. Mirrors `_compress_function_ast`. + fn compress_function_ast(&self, node: Node) -> String { + let start_row = node.start_position().row; + let end_row = node.end_position().row; + let node_lines: Vec<&str> = self.code_lines[start_row..=end_row].to_vec(); + let node_text = node_lines.join("\n"); + + let func_name = get_definition_name(node, self.code); + let body_limit = get_body_limit( + func_name.as_deref(), + self.body_limits, + self.config.max_body_lines, + ); + + if node_lines.len() as i64 <= body_limit + 2 { + return node_text; + } + + // Find the body node. + let mut body_node: Option = None; + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if self.lang.is_body(child.kind()) { + body_node = Some(child); + break; + } + } + let Some(body_node) = body_node else { + return node_text; + }; + + let node_start_line = start_row; + let body_start_line = body_node.start_position().row; + let body_end_line = body_node.end_position().row; + let sig_end = body_start_line - node_start_line; // exclusive + let body_end_rel = body_end_line - node_start_line + 1; // inclusive + + let signature_lines: Vec<&str>; + let mut body_lines: Vec<&str>; + let after_lines: Vec<&str>; + let brace_in_signature: bool; + + if sig_end == 0 && !self.lang.uses_colon_after_signature { + let first_line = node_lines[0]; + signature_lines = vec![first_line.trim_end()]; + body_lines = node_lines[1..body_end_rel].to_vec(); + after_lines = node_lines[body_end_rel..].to_vec(); + brace_in_signature = true; + } else { + signature_lines = node_lines[..sig_end].to_vec(); + body_lines = node_lines[sig_end..body_end_rel].to_vec(); + after_lines = node_lines[body_end_rel..].to_vec(); + brace_in_signature = false; + } + + // Brace detection for non-colon languages. + let mut opening_brace_line: Option<&str> = None; + let mut closing_brace_line: Option<&str> = None; + if !self.lang.uses_colon_after_signature { + if brace_in_signature { + // opening brace already in signature line. + } else if body_lines + .first() + .is_some_and(|l| l.trim_start().starts_with('{')) + { + opening_brace_line = Some(body_lines[0]); + body_lines = body_lines[1..].to_vec(); + } + if body_lines + .last() + .is_some_and(|l| l.trim_end().ends_with('}')) + { + closing_brace_line = Some(body_lines[body_lines.len() - 1]); + body_lines = body_lines[..body_lines.len() - 1].to_vec(); + } + } + + // Python docstring handling via AST. + let mut docstring_text = String::new(); + let mut ds_skip_lines: usize = 0; + if self.language == CodeLanguage::Python && body_node.child_count() > 0 { + let first_child = body_node.child(0).unwrap(); + // tree-sitter Python represents a docstring either as a bare + // `string` node in the block or as an `expression_statement` + // wrapping a `string`. Both map to the same docstring node. + let mut ds_node: Option = None; + if first_child.kind() == "string" + || (first_child.kind() == "expression_statement" + && first_child.child_count() > 0 + && first_child.child(0).unwrap().kind() == "string") + { + ds_node = Some(first_child); + } + if let Some(ds_node) = ds_node { + let ds_lines_count = ds_node.end_position().row - ds_node.start_position().row + 1; + let ds_start_rel = ds_node.start_position().row - body_node.start_position().row; + + match self.config.docstring_mode { + DocstringMode::Full => { + let endi = (ds_start_rel + ds_lines_count).min(body_lines.len()); + if ds_start_rel < body_lines.len() { + docstring_text = body_lines[ds_start_rel..endi].join("\n"); + } + } + DocstringMode::FirstLine => { + if ds_lines_count == 1 { + if let Some(l) = body_lines.get(ds_start_rel) { + docstring_text = (*l).to_string(); + } + } else if let Some(first_ds_line) = body_lines.get(ds_start_rel).copied() { + docstring_text = + first_line_docstring(first_ds_line, &body_lines, ds_start_rel); + } + } + DocstringMode::Remove | DocstringMode::None => {} + } + ds_skip_lines = ds_start_rel + ds_lines_count; + } + } + + // Statement-based body truncation. + let indent = if !body_lines.is_empty() { + detect_indent(&body_lines) + } else { + " ".to_string() + }; + + let mut ds_end_row: i64 = -1; + if ds_skip_lines > 0 && body_node.child_count() > 0 { + ds_end_row = (body_node.start_position().row + ds_skip_lines) as i64 - 1; + } + + const SKIP_TYPES: &[&str] = &[ + "{", + "}", + ";", + ",", + "comment", + "line_comment", + "block_comment", + ]; + + let mut body_stmts: Vec<(usize, usize)> = Vec::new(); + let mut bcursor = body_node.walk(); + for child in body_node.children(&mut bcursor) { + if (child.start_position().row as i64) <= ds_end_row { + continue; + } + if SKIP_TYPES.contains(&child.kind()) { + continue; + } + if !child.is_named() { + continue; + } + body_stmts.push((child.start_position().row, child.end_position().row)); + } + + let total_body_lines_count: i64 = + body_stmts.iter().map(|(s, e)| (*e - *s + 1) as i64).sum(); + + let mut kept_lines: Vec<&str> = Vec::new(); + let mut kept_line_count: i64 = 0; + for (s_row, e_row) in &body_stmts { + let stmt_lines: Vec<&str> = self.code_lines[*s_row..=*e_row].to_vec(); + let stmt_line_count = stmt_lines.len() as i64; + // `!kept_lines.is_empty()` == Python's `stmts_kept > 0` guard: + // always keep at least the first statement. + if kept_line_count + stmt_line_count > body_limit && !kept_lines.is_empty() { + break; + } + kept_lines.extend(stmt_lines); + kept_line_count += stmt_line_count; + } + + let omitted_lines = total_body_lines_count - kept_line_count; + + // Assemble. + let mut result_parts: Vec = Vec::new(); + if !signature_lines.is_empty() { + result_parts.extend(signature_lines.iter().map(|s| s.to_string())); + } else { + let sig_text = self.code[node.start_byte()..body_node.start_byte()].trim_end(); + result_parts.push(sig_text.to_string()); + } + if let Some(ob) = opening_brace_line { + result_parts.push(ob.to_string()); + } + if !docstring_text.is_empty() + && self.config.docstring_mode != DocstringMode::None + && self.config.docstring_mode != DocstringMode::Remove + { + result_parts.push(docstring_text); + } + if !kept_lines.is_empty() { + result_parts.extend(kept_lines.iter().map(|s| s.to_string())); + } + if omitted_lines > 0 { + result_parts.push(make_omitted_comment( + func_name.as_deref(), + omitted_lines, + &indent, + self.lang.comment_prefix, + self.analysis, + )); + if self.lang.uses_colon_after_signature { + result_parts.push(format!("{indent}pass")); + } + } + if let Some(cb) = closing_brace_line { + result_parts.push(cb.to_string()); + } else if !after_lines.is_empty() { + result_parts.extend(after_lines.iter().map(|s| s.to_string())); + } + + result_parts.join("\n") + } + + /// Compress a class by compressing each method individually. Mirrors + /// `_compress_class_ast`. + fn compress_class_ast(&self, node: Node) -> String { + let start_row = node.start_position().row; + let end_row = node.end_position().row; + let node_lines: Vec<&str> = self.code_lines[start_row..=end_row].to_vec(); + let node_text = node_lines.join("\n"); + + let mut body_node: Option = None; + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if self.lang.is_body(child.kind()) { + body_node = Some(child); + break; + } + } + let Some(body_node) = body_node else { + return node_text; + }; + + let node_start_line = start_row; + let body_start_line = body_node.start_position().row; + let sig_end = body_start_line - node_start_line; + let header_lines: Vec<&str> = if sig_end > 0 { + node_lines[..sig_end].to_vec() + } else { + vec![node_lines[0]] + }; + + let mut body_parts: Vec = Vec::new(); + let mut bcursor = body_node.walk(); + for child in body_node.children(&mut bcursor) { + let ck = child.kind(); + let child_start = child.start_position().row; + let child_end = child.end_position().row; + let child_text = self.lines_joined(child_start, child_end); + + if self.lang.is_function(ck) { + body_parts.push(self.compress_function_ast(child)); + } else if self.lang.decorator_node == Some(ck) { + let mut decorator_lines: Vec = Vec::new(); + let mut method_compressed: Option = None; + let mut dc = child.walk(); + for deco_child in child.children(&mut dc) { + if deco_child.kind() == "decorator" { + decorator_lines.push(self.node_text(deco_child).to_string()); + } else if self.lang.is_function(deco_child.kind()) { + method_compressed = Some(self.compress_function_ast(deco_child)); + } + } + match method_compressed { + Some(m) if !decorator_lines.is_empty() => { + body_parts.push(format!("{}\n{}", decorator_lines.join("\n"), m)); + } + Some(m) => body_parts.push(m), + None => body_parts.push(child_text), + } + } else if self.lang.is_class(ck) { + body_parts.push(self.compress_class_ast(child)); + } else if !child_text.trim().is_empty() { + body_parts.push(child_text); + } + } + + let mut result_parts: Vec = header_lines.iter().map(|s| s.to_string()).collect(); + result_parts.extend(body_parts); + + let body_end_line = body_node.end_position().row; + let body_end_rel = body_end_line - node_start_line + 1; + let after_lines: Vec<&str> = node_lines[body_end_rel..].to_vec(); + if !after_lines.is_empty() { + result_parts.extend(after_lines.iter().map(|s| s.to_string())); + } else if !self.lang.uses_colon_after_signature { + let last_body_line = node_lines.last().copied().unwrap_or(""); + if last_body_line.trim() == "}" { + result_parts.push(last_body_line.to_string()); + } + } + + result_parts.join("\n") + } +} + +/// FIRST_LINE multi-line docstring reconstruction. Mirrors the inner block +/// of `_compress_function_ast` (DocstringMode.FIRST_LINE, multi-line). +fn first_line_docstring(first_ds_line: &str, body_lines: &[&str], ds_start_rel: usize) -> String { + let ds_indent = leading_ws(first_ds_line); + let stripped = first_ds_line.trim(); + + const OPENERS: &[&str] = &["r\"\"\"", "r'''", "\"\"\"", "'''"]; + let mut quote = "\"\"\""; + let mut content_start = 0usize; + for opener in OPENERS { + if stripped.starts_with(opener) { + quote = &opener[opener.len() - 3..]; + content_start = opener.len(); + break; + } + } + + let mut first_content = stripped[content_start..].trim().to_string(); + for q in ["\"\"\"", "'''"] { + if first_content.ends_with(q) { + first_content = first_content[..first_content.len() - q.len()] + .trim() + .to_string(); + } + } + + if !first_content.is_empty() { + let prefix_part = &stripped[..content_start]; + format!("{ds_indent}{prefix_part}{first_content}{quote}") + } else if ds_start_rel + 1 < body_lines.len() { + let mut second_line = body_lines[ds_start_rel + 1].trim().to_string(); + for q in ["\"\"\"", "'''"] { + if second_line.ends_with(q) { + second_line = second_line[..second_line.len() - q.len()] + .trim() + .to_string(); + } + } + if !second_line.is_empty() { + format!("{ds_indent}{quote}{second_line}{quote}") + } else { + first_ds_line.to_string() + } + } else { + first_ds_line.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn py_round_int_is_half_to_even() { + // Reference: CPython round(x) — ties to even. + assert_eq!(py_round_int(0.5), 0); + assert_eq!(py_round_int(1.5), 2); + assert_eq!(py_round_int(2.5), 2); + assert_eq!(py_round_int(3.5), 4); + assert_eq!(py_round_int(-2.5), -2); + assert_eq!(py_round_int(2.4), 2); + assert_eq!(py_round_int(2.6), 3); + assert_eq!(py_round_int(0.0), 0); + assert_eq!(py_round_int(4.5), 4); + } + + #[test] + fn py_round3_matches_cpython() { + // Reference: CPython round(x, 3) — correctly rounded, ties to even. + assert_eq!(py_round3(1.0 / 3.0), 0.333); + assert_eq!(py_round3(2.0 / 3.0), 0.667); + assert_eq!(py_round3(0.0625), 0.062); // half-even (not 0.063) + assert_eq!(py_round3(0.1235), 0.123); + assert_eq!(py_round3(0.5005), 0.5); + assert_eq!(py_round3(0.12345), 0.123); + assert_eq!(py_round3(1.0), 1.0); + assert_eq!(py_round3(0.524822695035461), 0.525); + } + + #[test] + fn estimate_tokens_uses_chars_div_4_min_1() { + assert_eq!(estimate_tokens("abcd"), 1); + assert_eq!(estimate_tokens("abcdefgh"), 2); + assert_eq!(estimate_tokens("a"), 1); // max(1, 0) + } + + #[test] + fn detect_language_basic() { + let (lang, conf) = detect_language("import os\n\ndef f(x):\n return x + 1\n"); + assert_eq!(lang, CodeLanguage::Python); + assert!(conf >= 0.3 && conf <= 1.0); + + let (lang, _) = detect_language("package main\n\nfunc main() {}\n"); + assert_eq!(lang, CodeLanguage::Go); + + let (lang, conf) = detect_language(""); + assert_eq!(lang, CodeLanguage::Unknown); + assert_eq!(conf, 0.0); + + let (lang, _) = detect_language("just plain english prose with no code here at all"); + assert_eq!(lang, CodeLanguage::Unknown); + } + + #[test] + fn empty_and_short_passthrough() { + let c = CodeAwareCompressor::new(CodeCompressorConfig::default()); + let r = c.compress(""); + assert_eq!(r.compressed, ""); + assert_eq!(r.original_tokens, 0); + assert_eq!(r.language, CodeLanguage::Unknown); + + let r = c.compress("def f(): pass"); + assert_eq!(r.compressed, "def f(): pass"); // < min_tokens → passthrough + assert_eq!(r.compression_ratio, 1.0); + assert_eq!(r.language, CodeLanguage::Unknown); + } +} diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rs index 3d0525acd..2ec4bf588 100644 --- a/crates/headroom-core/src/transforms/mod.rs +++ b/crates/headroom-core/src/transforms/mod.rs @@ -17,6 +17,7 @@ pub mod adaptive_sizer; pub mod anchor_selector; +pub mod code_compressor; pub mod content_detector; pub mod detection; pub mod diff_compressor; @@ -35,6 +36,10 @@ pub mod tag_protector; pub mod text_crusher; pub mod unidiff_detector; +pub use code_compressor::{ + detect_language, CodeAwareCompressor, CodeCompressionResult, CodeCompressorConfig, + CodeLanguage, DocstringMode, +}; pub use content_detector::{ detect_content_type, is_json_array_of_dicts, ContentType, DetectionResult, }; diff --git a/crates/headroom-core/tests/code_compressor_parity.rs b/crates/headroom-core/tests/code_compressor_parity.rs new file mode 100644 index 000000000..86fc6502d --- /dev/null +++ b/crates/headroom-core/tests/code_compressor_parity.rs @@ -0,0 +1,150 @@ +//! Byte-parity integration test for the CodeCompressor Rust port. +//! +//! Runs the production [`CodeAwareCompressor`] against the fixtures recorded +//! from the Python reference (`tests/parity/fixtures/code_aware_compressor/`) +//! and asserts the serialized result matches field-for-field. +//! +//! Unlike the Kompress test, this needs no model/network: the per-language +//! tree-sitter grammars are compiled into the crate. Grammar-version parity +//! is guaranteed by the exact Cargo pins matching the Python wheels the +//! fixtures were recorded against (see `Cargo.toml`). + +use std::fs; +use std::path::{Path, PathBuf}; + +use headroom_core::transforms::code_compressor::{ + CodeAwareCompressor, CodeCompressorConfig, DocstringMode, +}; +use serde_json::Value; + +fn config_from_fixture(config: &Value) -> CodeCompressorConfig { + let d = CodeCompressorConfig::default(); + CodeCompressorConfig { + preserve_imports: config + .get("preserve_imports") + .and_then(|v| v.as_bool()) + .unwrap_or(d.preserve_imports), + preserve_signatures: config + .get("preserve_signatures") + .and_then(|v| v.as_bool()) + .unwrap_or(d.preserve_signatures), + preserve_type_annotations: config + .get("preserve_type_annotations") + .and_then(|v| v.as_bool()) + .unwrap_or(d.preserve_type_annotations), + preserve_decorators: config + .get("preserve_decorators") + .and_then(|v| v.as_bool()) + .unwrap_or(d.preserve_decorators), + docstring_mode: config + .get("docstring_mode") + .and_then(|v| v.as_str()) + .and_then(DocstringMode::from_value) + .unwrap_or(d.docstring_mode), + target_compression_rate: config + .get("target_compression_rate") + .and_then(|v| v.as_f64()) + .unwrap_or(d.target_compression_rate), + max_body_lines: config + .get("max_body_lines") + .and_then(|v| v.as_i64()) + .unwrap_or(d.max_body_lines), + compress_comments: config + .get("compress_comments") + .and_then(|v| v.as_bool()) + .unwrap_or(d.compress_comments), + min_tokens_for_compression: config + .get("min_tokens_for_compression") + .and_then(|v| v.as_i64()) + .unwrap_or(d.min_tokens_for_compression), + language_hint: config + .get("language_hint") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + fallback_to_kompress: config + .get("fallback_to_kompress") + .and_then(|v| v.as_bool()) + .unwrap_or(d.fallback_to_kompress), + semantic_analysis: config + .get("semantic_analysis") + .and_then(|v| v.as_bool()) + .unwrap_or(d.semantic_analysis), + enable_ccr: config + .get("enable_ccr") + .and_then(|v| v.as_bool()) + .unwrap_or(d.enable_ccr), + ccr_ttl: config + .get("ccr_ttl") + .and_then(|v| v.as_i64()) + .unwrap_or(d.ccr_ttl), + } +} + +#[test] +fn code_compressor_matches_python_fixtures_byte_for_byte() { + let fixtures_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/parity/fixtures/code_aware_compressor"); + assert!( + fixtures_dir.exists(), + "fixtures dir {} missing — run scripts/record_code_compressor_fixtures.py", + fixtures_dir.display() + ); + + let mut paths: Vec = fs::read_dir(&fixtures_dir) + .expect("read fixtures dir") + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.extension().map(|x| x == "json").unwrap_or(false)) + .collect(); + paths.sort(); + + let mut checked = 0usize; + let mut nontrivial = 0usize; + for path in &paths { + let fx: Value = serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); + let name = path.file_name().unwrap().to_string_lossy().to_string(); + let content = fx["input"].as_str().expect("fixture.input string"); + let expected = &fx["output"]; + + let cfg = config_from_fixture(&fx["config"]); + let result = CodeAwareCompressor::new(cfg).compress(content); + + let mut symbol_scores = serde_json::Map::new(); + for (k, v) in &result.symbol_scores { + symbol_scores.insert(k.clone(), serde_json::json!(v)); + } + let actual = serde_json::json!({ + "cache_key": result.cache_key, + "compressed": result.compressed, + "compressed_bodies": result.compressed_bodies, + "compressed_tokens": result.compressed_tokens, + "compression_ratio": result.compression_ratio, + "language": result.language.value(), + "language_confidence": result.language_confidence, + "original": result.original, + "original_tokens": result.original_tokens, + "preserved_imports": result.preserved_imports, + "preserved_signatures": result.preserved_signatures, + "symbol_scores": serde_json::Value::Object(symbol_scores), + "syntax_valid": result.syntax_valid, + }); + // Normalize through serde (f64 round-trip), matching the harness. + let actual: Value = serde_json::from_str(&serde_json::to_string(&actual).unwrap()).unwrap(); + + assert_eq!( + &actual, expected, + "[{name}] code compressor output diverged from Python reference" + ); + if result.compressed != content { + nontrivial += 1; + } + checked += 1; + } + + assert!(checked >= 20, "expected >= 20 fixtures, got {checked}"); + assert!( + nontrivial >= 10, + "expected >= 10 non-trivial compressions, got {nontrivial} (are the fixtures all passthroughs?)" + ); + eprintln!("code_compressor parity: {checked} fixtures matched ({nontrivial} non-trivial)"); +} diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs index 694ab6c33..a3d4c101d 100644 --- a/crates/headroom-parity/src/lib.rs +++ b/crates/headroom-parity/src/lib.rs @@ -713,6 +713,124 @@ impl TransformComparator for KompressComparator { } } +/// Real comparator for the `code_aware_compressor` transform. Drives the +/// Rust AST code compressor over the recorded fixture inputs and emits the +/// same shape Python's recorder serializes for `CodeCompressionResult` +/// (dataclass fields via `asdict`; the `@property` derivatives are not +/// serialized). Fixtures are recorded with `enable_ccr=False` and +/// `fallback_to_kompress=False` so the output is deterministic and +/// store/model-independent. +/// +/// Grammar-version parity is the precondition: the Rust `tree-sitter-` +/// crates are pinned to the exact versions of the Python wheels the fixtures +/// were recorded against (see `headroom-core/Cargo.toml`). +pub struct CodeCompressorComparator; + +impl TransformComparator for CodeCompressorComparator { + fn name(&self) -> &str { + "code_aware_compressor" + } + + fn run( + &self, + input: &serde_json::Value, + config: &serde_json::Value, + ) -> Result { + use headroom_core::transforms::code_compressor::{ + CodeAwareCompressor, CodeCompressorConfig, DocstringMode, + }; + + let content = input + .as_str() + .context("code_aware_compressor fixture input must be a JSON string")?; + + let defaults = CodeCompressorConfig::default(); + let cfg = CodeCompressorConfig { + preserve_imports: config + .get("preserve_imports") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.preserve_imports), + preserve_signatures: config + .get("preserve_signatures") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.preserve_signatures), + preserve_type_annotations: config + .get("preserve_type_annotations") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.preserve_type_annotations), + preserve_decorators: config + .get("preserve_decorators") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.preserve_decorators), + docstring_mode: config + .get("docstring_mode") + .and_then(|v| v.as_str()) + .and_then(DocstringMode::from_value) + .unwrap_or(defaults.docstring_mode), + target_compression_rate: config + .get("target_compression_rate") + .and_then(|v| v.as_f64()) + .unwrap_or(defaults.target_compression_rate), + max_body_lines: config + .get("max_body_lines") + .and_then(|v| v.as_i64()) + .unwrap_or(defaults.max_body_lines), + compress_comments: config + .get("compress_comments") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.compress_comments), + min_tokens_for_compression: config + .get("min_tokens_for_compression") + .and_then(|v| v.as_i64()) + .unwrap_or(defaults.min_tokens_for_compression), + language_hint: config + .get("language_hint") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + fallback_to_kompress: config + .get("fallback_to_kompress") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.fallback_to_kompress), + semantic_analysis: config + .get("semantic_analysis") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.semantic_analysis), + enable_ccr: config + .get("enable_ccr") + .and_then(|v| v.as_bool()) + .unwrap_or(defaults.enable_ccr), + ccr_ttl: config + .get("ccr_ttl") + .and_then(|v| v.as_i64()) + .unwrap_or(defaults.ccr_ttl), + }; + + let compressor = CodeAwareCompressor::new(cfg); + let result = compressor.compress(content); + + let mut symbol_scores = serde_json::Map::new(); + for (name, score) in &result.symbol_scores { + symbol_scores.insert(name.clone(), serde_json::json!(score)); + } + + Ok(serde_json::json!({ + "cache_key": result.cache_key, + "compressed": result.compressed, + "compressed_bodies": result.compressed_bodies, + "compressed_tokens": result.compressed_tokens, + "compression_ratio": result.compression_ratio, + "language": result.language.value(), + "language_confidence": result.language_confidence, + "original": result.original, + "original_tokens": result.original_tokens, + "preserved_imports": result.preserved_imports, + "preserved_signatures": result.preserved_signatures, + "symbol_scores": serde_json::Value::Object(symbol_scores), + "syntax_valid": result.syntax_valid, + })) + } +} + /// Every built-in comparator, in a stable order. pub fn builtin_comparators() -> Vec> { vec![ @@ -725,6 +843,7 @@ pub fn builtin_comparators() -> Vec> { Box::new(ContentDetectorComparator), Box::new(TextCrusherComparator), Box::new(KompressComparator::new()), + Box::new(CodeCompressorComparator), ] } diff --git a/scripts/record_code_compressor_fixtures.py b/scripts/record_code_compressor_fixtures.py new file mode 100644 index 000000000..819192f2f --- /dev/null +++ b/scripts/record_code_compressor_fixtures.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Record standard parity fixtures for the CodeAwareCompressor only. + +Installs the individual-grammar parser patch, then drives the Python +`CodeAwareCompressor` (enable_ccr=False, fallback_to_kompress=False) over +`_varied_code_inputs()` while `record_all()` has the `compress` method +patched, so only `tests/parity/fixtures/code_aware_compressor/` is +(re)written — no churn to other transforms' fixtures. + +The grammar wheels must be installed at the versions the Rust crates pin +(same version number on PyPI + crates.io = same grammar source = identical +ASTs; verified by the grammar-parity canary): + + pip install tree-sitter==0.25.2 \\ + tree-sitter-python==0.25.0 tree-sitter-javascript==0.25.0 \\ + tree-sitter-typescript==0.23.2 tree-sitter-go==0.25.0 \\ + tree-sitter-rust==0.24.2 tree-sitter-java==0.23.5 \\ + tree-sitter-c==0.24.2 tree-sitter-cpp==0.23.4 + python scripts/record_code_compressor_fixtures.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO)) + + +def main() -> int: + from tests.parity.recorder import ( + _docstring_mode_inputs, + _varied_code_inputs, + install_individual_grammar_parsers, + record_all, + ) + + statuses = record_all() + if not statuses.get("code_aware_compressor", "").startswith("patched"): + print( + f"code_aware_compressor not patched: {statuses.get('code_aware_compressor')}", + file=sys.stderr, + ) + return 1 + + install_individual_grammar_parsers() + + from headroom.transforms.code_compressor import ( + CodeAwareCompressor, + CodeCompressorConfig, + DocstringMode, + ) + + inputs = _varied_code_inputs() + cac = CodeAwareCompressor(CodeCompressorConfig(enable_ccr=False, fallback_to_kompress=False)) + for s in inputs: + cac.compress(s) + + # Non-default docstring modes (FULL / REMOVE) over docstring-bearing + # samples — distinct config hash → distinct fixtures. + ds_inputs = _docstring_mode_inputs() + extra = 0 + for mode in (DocstringMode.FULL, DocstringMode.REMOVE): + c = CodeAwareCompressor( + CodeCompressorConfig(enable_ccr=False, fallback_to_kompress=False, docstring_mode=mode) + ) + for s in ds_inputs: + c.compress(s) + extra += 1 + + out_dir = REPO / "tests" / "parity" / "fixtures" / "code_aware_compressor" + n = len(list(out_dir.glob("*.json"))) + print( + f"recorded {n} code_aware_compressor fixtures " + f"from {len(inputs)} default + {extra} docstring-mode inputs -> {out_dir}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/parity/fixtures/code_aware_compressor/07dd1a61089c1e53.json b/tests/parity/fixtures/code_aware_compressor/07dd1a61089c1e53.json new file mode 100644 index 000000000..9496f3d65 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/07dd1a61089c1e53.json @@ -0,0 +1,41 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List process(List items) {\n List results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n", + "input_sha256": "07dd1a61089c1e5326d1d4d5a404a5363fc3f45db1b46bd4554987916df4d575", + "output": { + "cache_key": null, + "compressed": "package com.example;\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List process(List items) {\n List results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}", + "compressed_bodies": 0, + "compressed_tokens": 180, + "compression_ratio": 1.0, + "language": "java", + "language_confidence": 1.0, + "original": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List process(List items) {\n List results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n", + "original_tokens": 180, + "preserved_imports": 3, + "preserved_signatures": 0, + "symbol_scores": { + "Processor": 0.071, + "String": 1.0, + "process": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.332197+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/095be1adad76b588.json b/tests/parity/fixtures/code_aware_compressor/095be1adad76b588.json new file mode 100644 index 000000000..70093ad6e --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/095be1adad76b588.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "#include \n#include \n#include \n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n", + "input_sha256": "095be1adad76b588f996e2c39b843e28ef41ec58b2ca8bb963c4fe9389027413", + "output": { + "cache_key": null, + "compressed": "#include \n\n#include \n\n#include \n\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n // [8 lines omitted]\n}\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}", + "compressed_bodies": 0, + "compressed_tokens": 93, + "compression_ratio": 0.7045454545454546, + "language": "cpp", + "language_confidence": 1.0, + "original": "#include \n#include \n#include \n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n", + "original_tokens": 132, + "preserved_imports": 3, + "preserved_signatures": 2, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.333831+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/1e319c5e33af4af1.json b/tests/parity/fixtures/code_aware_compressor/1e319c5e33af4af1.json new file mode 100644 index 000000000..e0dee7cfa --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/1e319c5e33af4af1.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "const a = 1;\nfunction g() { return a; }\n", + "input_sha256": "1e319c5e33af4af1aca7a0b5d4fe402d7efe7c4adfcce6353fe85dd55d26d7eb", + "output": { + "cache_key": null, + "compressed": "const a = 1;\nfunction g() { return a; }\n", + "compressed_bodies": 0, + "compressed_tokens": 10, + "compression_ratio": 1.0, + "language": "unknown", + "language_confidence": 0.0, + "original": "const a = 1;\nfunction g() { return a; }\n", + "original_tokens": 10, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.336456+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/1e496ebaf89fab96.json b/tests/parity/fixtures/code_aware_compressor/1e496ebaf89fab96.json new file mode 100644 index 000000000..af850b86f --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/1e496ebaf89fab96.json @@ -0,0 +1,42 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "remove", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n", + "input_sha256": "1e496ebaf89fab96ac4f3836146f87ea6830de66a27cbf4134068ad07be50b74", + "output": { + "cache_key": null, + "compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))", + "compressed_bodies": 0, + "compressed_tokens": 129, + "compression_ratio": 0.6292682926829268, + "language": "python", + "language_confidence": 1.0, + "original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n", + "original_tokens": 205, + "preserved_imports": 3, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 1.0, + "process": 0.5, + "reset": 0.0, + "standalone": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.358515+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/2c239d4af41282c2.json b/tests/parity/fixtures/code_aware_compressor/2c239d4af41282c2.json new file mode 100644 index 000000000..d64dd62e2 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/2c239d4af41282c2.json @@ -0,0 +1,41 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n", + "input_sha256": "2c239d4af41282c27453261058252f76b9f173d4b16d69fa1094010556969416", + "output": { + "cache_key": null, + "compressed": "import json\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n # [6 lines omitted]\n pass\ndef transform(records, factor):\n out = []\n # [6 lines omitted]\n pass\ndef run(path, factor):\n data = load(path)\n # [3 lines omitted; calls: load, transform]\n pass", + "compressed_bodies": 0, + "compressed_tokens": 74, + "compression_ratio": 0.524822695035461, + "language": "python", + "language_confidence": 1.0, + "original": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n", + "original_tokens": 141, + "preserved_imports": 1, + "preserved_signatures": 3, + "symbol_scores": { + "load": 1.0, + "run": 0.0, + "transform": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.321701+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/2d214f401618d639.json b/tests/parity/fixtures/code_aware_compressor/2d214f401618d639.json new file mode 100644 index 000000000..eed899883 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/2d214f401618d639.json @@ -0,0 +1,41 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n\n# variant 1", + "input_sha256": "2d214f401618d6392d0d152fc1446571e6f28045d51569b98af9bf59841eb969", + "output": { + "cache_key": null, + "compressed": "import json\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n # [6 lines omitted]\n pass\ndef transform(records, factor):\n out = []\n # [6 lines omitted]\n pass\ndef run(path, factor):\n data = load(path)\n # [3 lines omitted; calls: load, transform]\n pass\n\n# variant 1", + "compressed_bodies": 0, + "compressed_tokens": 77, + "compression_ratio": 0.5347222222222222, + "language": "python", + "language_confidence": 1.0, + "original": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n\n# variant 1", + "original_tokens": 144, + "preserved_imports": 1, + "preserved_signatures": 3, + "symbol_scores": { + "load": 1.0, + "run": 0.0, + "transform": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.353260+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/36e5ff5093790e6f.json b/tests/parity/fixtures/code_aware_compressor/36e5ff5093790e6f.json new file mode 100644 index 000000000..df2941ca5 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/36e5ff5093790e6f.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "def f(x):\n return x + 1\n", + "input_sha256": "36e5ff5093790e6fc3473727b0f49b944a68d1e9b0417ac97c4d0c18536d4d96", + "output": { + "cache_key": null, + "compressed": "def f(x):\n return x + 1\n", + "compressed_bodies": 0, + "compressed_tokens": 6, + "compression_ratio": 1.0, + "language": "unknown", + "language_confidence": 0.0, + "original": "def f(x):\n return x + 1\n", + "original_tokens": 6, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.336336+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/3e706cfb03aa3c8e.json b/tests/parity/fixtures/code_aware_compressor/3e706cfb03aa3c8e.json new file mode 100644 index 000000000..2a224eca0 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/3e706cfb03aa3c8e.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n\n// variant 1", + "input_sha256": "3e706cfb03aa3c8edd80d11d4f3ce508aba524ac79f5e2f7647bc28c0ec01217", + "output": { + "cache_key": null, + "compressed": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 150, + "compression_ratio": 1.0, + "language": "javascript", + "language_confidence": 1.0, + "original": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n\n// variant 1", + "original_tokens": 150, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.340479+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/411d5549bcd5f50d.json b/tests/parity/fixtures/code_aware_compressor/411d5549bcd5f50d.json new file mode 100644 index 000000000..827d047a7 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/411d5549bcd5f50d.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "#include \n#include \n#include \n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n\n// variant 1", + "input_sha256": "411d5549bcd5f50dc15031bec587e643ea032e4975c90fb36d6fe7ed79195b94", + "output": { + "cache_key": null, + "compressed": "#include \n\n#include \n\n#include \n\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n // [8 lines omitted]\n}\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 96, + "compression_ratio": 0.7111111111111111, + "language": "cpp", + "language_confidence": 1.0, + "original": "#include \n#include \n#include \n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n\n// variant 1", + "original_tokens": 135, + "preserved_imports": 3, + "preserved_signatures": 2, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.348562+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/420dd19a087526ca.json b/tests/parity/fixtures/code_aware_compressor/420dd19a087526ca.json new file mode 100644 index 000000000..56d24041e --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/420dd19a087526ca.json @@ -0,0 +1,39 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "#include \n#include \n#include \n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n", + "input_sha256": "420dd19a087526ca88d1cc71110894f73029cee57533b406ed005b382200273b", + "output": { + "cache_key": null, + "compressed": "#include \n\n#include \n\n#include \n\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n}\n// namespace app", + "compressed_bodies": 0, + "compressed_tokens": 278, + "compression_ratio": 1.759493670886076, + "language": "cpp", + "language_confidence": 1.0, + "original": "#include \n#include \n#include \n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n", + "original_tokens": 158, + "preserved_imports": 3, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 0.5 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.336058+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/53030c7c11ce0082.json b/tests/parity/fixtures/code_aware_compressor/53030c7c11ce0082.json new file mode 100644 index 000000000..a59d2c40f --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/53030c7c11ce0082.json @@ -0,0 +1,42 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "full", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n", + "input_sha256": "53030c7c11ce0082b64002c5a4d5b920039825459494a5ee3dd09dd48b040b34", + "output": { + "cache_key": null, + "compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))", + "compressed_bodies": 0, + "compressed_tokens": 145, + "compression_ratio": 0.7073170731707317, + "language": "python", + "language_confidence": 1.0, + "original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n", + "original_tokens": 205, + "preserved_imports": 3, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 1.0, + "process": 0.5, + "reset": 0.0, + "standalone": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.355261+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/5c057f7bfd56af05.json b/tests/parity/fixtures/code_aware_compressor/5c057f7bfd56af05.json new file mode 100644 index 000000000..e472a6fe4 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/5c057f7bfd56af05.json @@ -0,0 +1,41 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List process(List items) {\n List results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n\n// variant 1", + "input_sha256": "5c057f7bfd56af058ab2f8745bf0242d97e3a458ce8efedae8263066c1c709a7", + "output": { + "cache_key": null, + "compressed": "package com.example;\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List process(List items) {\n List results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 183, + "compression_ratio": 1.0, + "language": "java", + "language_confidence": 1.0, + "original": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List process(List items) {\n List results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n\n// variant 1", + "original_tokens": 183, + "preserved_imports": 3, + "preserved_signatures": 0, + "symbol_scores": { + "Processor": 0.071, + "String": 1.0, + "process": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.346983+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/6c04d398f9c3014e.json b/tests/parity/fixtures/code_aware_compressor/6c04d398f9c3014e.json new file mode 100644 index 000000000..067415e3b --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/6c04d398f9c3014e.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "remove", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "input_sha256": "6c04d398f9c3014eca3854ea1146e9f3ce76f3611b3cabc7170347e2c550200f", + "output": { + "cache_key": null, + "compressed": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "compressed_bodies": 0, + "compressed_tokens": 82, + "compression_ratio": 1.0, + "language": "unknown", + "language_confidence": 0.0, + "original": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "original_tokens": 82, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:12:54.716311+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/716a5a2380907e71.json b/tests/parity/fixtures/code_aware_compressor/716a5a2380907e71.json new file mode 100644 index 000000000..d202c6ea2 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/716a5a2380907e71.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n\n// variant 1", + "input_sha256": "716a5a2380907e7165334f0d0ce4c4502d6756647f8cb24dacf9b4882c1f8ede", + "output": { + "cache_key": null, + "compressed": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 120, + "compression_ratio": 1.0, + "language": "go", + "language_confidence": 1.0, + "original": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n\n// variant 1", + "original_tokens": 120, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.342039+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/791805bff164ee9d.json b/tests/parity/fixtures/code_aware_compressor/791805bff164ee9d.json new file mode 100644 index 000000000..b28180afa --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/791805bff164ee9d.json @@ -0,0 +1,40 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "remove", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n", + "input_sha256": "791805bff164ee9d19721a22676e34ccb73c8ad3439f93225be2946d43140138", + "output": { + "cache_key": null, + "compressed": "import logging\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n logger = logging.getLogger()\n # [9 lines omitted]\n pass\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n # [1 lines omitted]\n pass", + "compressed_bodies": 0, + "compressed_tokens": 82, + "compression_ratio": 0.3761467889908257, + "language": "python", + "language_confidence": 1.0, + "original": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n", + "original_tokens": 218, + "preserved_imports": 1, + "preserved_signatures": 2, + "symbol_scores": { + "configure": 0.5, + "teardown": 0.5 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.359509+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/7cee85d20b3604dd.json b/tests/parity/fixtures/code_aware_compressor/7cee85d20b3604dd.json new file mode 100644 index 000000000..a685ebc57 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/7cee85d20b3604dd.json @@ -0,0 +1,42 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n\n# variant 1", + "input_sha256": "7cee85d20b3604dd04b1354bc454db103725b7e76a2973b23a4b16c10800b629", + "output": { + "cache_key": null, + "compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n# variant 1", + "compressed_bodies": 0, + "compressed_tokens": 148, + "compression_ratio": 0.7115384615384616, + "language": "python", + "language_confidence": 1.0, + "original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n\n# variant 1", + "original_tokens": 208, + "preserved_imports": 3, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 1.0, + "process": 0.5, + "reset": 0.0, + "standalone": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.338646+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/88fe6b4b0a9908c5.json b/tests/parity/fixtures/code_aware_compressor/88fe6b4b0a9908c5.json new file mode 100644 index 000000000..0d462d3b7 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/88fe6b4b0a9908c5.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "full", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "input_sha256": "88fe6b4b0a9908c54185177fa5adfdd8533ececd54e57ba74391fd228ee685c4", + "output": { + "cache_key": null, + "compressed": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "compressed_bodies": 0, + "compressed_tokens": 82, + "compression_ratio": 1.0, + "language": "unknown", + "language_confidence": 0.0, + "original": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "original_tokens": 82, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:12:54.714085+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/921876ad5dd26cf6.json b/tests/parity/fixtures/code_aware_compressor/921876ad5dd26cf6.json new file mode 100644 index 000000000..a906619f0 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/921876ad5dd26cf6.json @@ -0,0 +1,40 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "from collections import defaultdict\n\n\ndef build_index(records):\n index = defaultdict(list)\n for rec in records:\n key = rec.get(\"id\")\n if key is None:\n continue\n index[key].append(rec)\n if len(index[key]) > 100:\n index[key] = index[key][:100]\n return index\n\n\ndef merge(a, b):\n out = dict(a)\n for k, v in b.items():\n out[k] = v\n return out\n", + "input_sha256": "921876ad5dd26cf623a97a7668ec4a1026f9f8b4e2d97af154943f29482c160c", + "output": { + "cache_key": null, + "compressed": "from collections import defaultdict\n\ndef build_index(records):\n index = defaultdict(list)\n # [8 lines omitted]\n pass\ndef merge(a, b):\n out = dict(a)\n # [3 lines omitted]\n pass", + "compressed_bodies": 0, + "compressed_tokens": 48, + "compression_ratio": 0.46601941747572817, + "language": "python", + "language_confidence": 1.0, + "original": "from collections import defaultdict\n\n\ndef build_index(records):\n index = defaultdict(list)\n for rec in records:\n key = rec.get(\"id\")\n if key is None:\n continue\n index[key].append(rec)\n if len(index[key]) > 100:\n index[key] = index[key][:100]\n return index\n\n\ndef merge(a, b):\n out = dict(a)\n for k, v in b.items():\n out[k] = v\n return out\n", + "original_tokens": 103, + "preserved_imports": 1, + "preserved_signatures": 2, + "symbol_scores": { + "build_index": 0.5, + "merge": 0.5 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.319542+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/b16202e9b81a5bba.json b/tests/parity/fixtures/code_aware_compressor/b16202e9b81a5bba.json new file mode 100644 index 000000000..7f424b25f --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/b16202e9b81a5bba.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n", + "input_sha256": "b16202e9b81a5bba3ca5c9ff0ecf632870f77938a98bb41c1f0026e57b403753", + "output": { + "cache_key": null, + "compressed": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n", + "compressed_bodies": 0, + "compressed_tokens": 147, + "compression_ratio": 1.0, + "language": "javascript", + "language_confidence": 1.0, + "original": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n", + "original_tokens": 147, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.323440+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/b753b4725da91f71.json b/tests/parity/fixtures/code_aware_compressor/b753b4725da91f71.json new file mode 100644 index 000000000..88087f186 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/b753b4725da91f71.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n", + "input_sha256": "b753b4725da91f71b39e71568539f9d307bf18184cc91ce2558d08865ac710f5", + "output": { + "cache_key": null, + "compressed": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n", + "compressed_bodies": 0, + "compressed_tokens": 117, + "compression_ratio": 1.0, + "language": "go", + "language_confidence": 1.0, + "original": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n", + "original_tokens": 117, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.327065+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/b8fa00d165cd059a.json b/tests/parity/fixtures/code_aware_compressor/b8fa00d165cd059a.json new file mode 100644 index 000000000..4fded5b59 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/b8fa00d165cd059a.json @@ -0,0 +1,42 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec) -> Vec {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n\n// variant 1", + "input_sha256": "b8fa00d165cd059a0549473920e4d61bcbde3980257d328f2cbc76fc3a3db787", + "output": { + "cache_key": null, + "compressed": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec) -> Vec {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n // [1 lines omitted; calls: Processor, new, process]\n}\n\n#[derive(Debug)]\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 185, + "compression_ratio": 1.022099447513812, + "language": "rust", + "language_confidence": 1.0, + "original": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec) -> Vec {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n\n// variant 1", + "original_tokens": 181, + "preserved_imports": 1, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 1.0, + "main": 0.0, + "new": 0.333, + "process": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.345389+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/ba9c0fda1beda6ec.json b/tests/parity/fixtures/code_aware_compressor/ba9c0fda1beda6ec.json new file mode 100644 index 000000000..54e0aa54b --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/ba9c0fda1beda6ec.json @@ -0,0 +1,39 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "#include \n#include \n#include \n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\n// variant 1", + "input_sha256": "ba9c0fda1beda6ec7b8447e7aa39a743f073cf02e914f52e99aaa6ebda3c4dc1", + "output": { + "cache_key": null, + "compressed": "#include \n\n#include \n\n#include \n\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n}\n// namespace app\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 281, + "compression_ratio": 1.7453416149068324, + "language": "cpp", + "language_confidence": 1.0, + "original": "#include \n#include \n#include \n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector process(const std::vector &items) {\n std::vector results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\n// variant 1", + "original_tokens": 161, + "preserved_imports": 3, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 0.5 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.350553+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/bd14bdd440e5a1ab.json b/tests/parity/fixtures/code_aware_compressor/bd14bdd440e5a1ab.json new file mode 100644 index 000000000..e205f4beb --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/bd14bdd440e5a1ab.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe = T | null;\n\nexport function lookup(users: User[], id: number): Maybe {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n\n// variant 1", + "input_sha256": "bd14bdd440e5a1ab8bef7b20c809ac1617a9be9f2ba8b700e7f518bfb474e7da", + "output": { + "cache_key": null, + "compressed": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe = T | null;\n\nexport function lookup(users: User[], id: number): Maybe {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n\n// variant 1", + "compressed_bodies": 0, + "compressed_tokens": 132, + "compression_ratio": 1.0, + "language": "typescript", + "language_confidence": 1.0, + "original": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe = T | null;\n\nexport function lookup(users: User[], id: number): Maybe {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n\n// variant 1", + "original_tokens": 132, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.352262+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/bfcb158739b34b43.json b/tests/parity/fixtures/code_aware_compressor/bfcb158739b34b43.json new file mode 100644 index 000000000..1214a6328 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/bfcb158739b34b43.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "input_sha256": "bfcb158739b34b4381a4c23e9ff84bbe04882bb43e6e658aaee4cb6865802fa2", + "output": { + "cache_key": null, + "compressed": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "compressed_bodies": 0, + "compressed_tokens": 82, + "compression_ratio": 1.0, + "language": "unknown", + "language_confidence": 0.0, + "original": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n", + "original_tokens": 82, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:12:54.677594+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/d72fde8ea01cb782.json b/tests/parity/fixtures/code_aware_compressor/d72fde8ea01cb782.json new file mode 100644 index 000000000..4f59ac1a9 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/d72fde8ea01cb782.json @@ -0,0 +1,40 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "full", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n", + "input_sha256": "d72fde8ea01cb782c6e3de2e9d8322b40cc5e27593eefbd1049b64ad4467c154", + "output": { + "cache_key": null, + "compressed": "import logging\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n # [9 lines omitted]\n pass\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n # [1 lines omitted]\n pass", + "compressed_bodies": 0, + "compressed_tokens": 148, + "compression_ratio": 0.6788990825688074, + "language": "python", + "language_confidence": 1.0, + "original": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n", + "original_tokens": 218, + "preserved_imports": 1, + "preserved_signatures": 2, + "symbol_scores": { + "configure": 0.5, + "teardown": 0.5 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.356466+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/da21f7240cb20e8b.json b/tests/parity/fixtures/code_aware_compressor/da21f7240cb20e8b.json new file mode 100644 index 000000000..920726f20 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/da21f7240cb20e8b.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "This is just a paragraph of plain English prose that contains no recognizable source code constructs at all, so the language detector should classify it as unknown and the compressor should pass it through unchanged without attempting any AST based compression here.\n", + "input_sha256": "da21f7240cb20e8b65bee5b06242e82302ace2ad72e430bbcdca59964d6e6274", + "output": { + "cache_key": null, + "compressed": "This is just a paragraph of plain English prose that contains no recognizable source code constructs at all, so the language detector should classify it as unknown and the compressor should pass it through unchanged without attempting any AST based compression here.\n", + "compressed_bodies": 0, + "compressed_tokens": 66, + "compression_ratio": 1.0, + "language": "unknown", + "language_confidence": 0.0, + "original": "This is just a paragraph of plain English prose that contains no recognizable source code constructs at all, so the language detector should classify it as unknown and the compressor should pass it through unchanged without attempting any AST based compression here.\n", + "original_tokens": 66, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.336562+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/e79625ba83a6f22d.json b/tests/parity/fixtures/code_aware_compressor/e79625ba83a6f22d.json new file mode 100644 index 000000000..efa23c54d --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/e79625ba83a6f22d.json @@ -0,0 +1,40 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n", + "input_sha256": "e79625ba83a6f22d3bd5d25aa779d38fa07291a4169a5d23fcf2e087d629121c", + "output": { + "cache_key": null, + "compressed": "import logging\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\"\"\"\n logger = logging.getLogger()\n # [9 lines omitted]\n pass\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n # [1 lines omitted]\n pass", + "compressed_bodies": 0, + "compressed_tokens": 96, + "compression_ratio": 0.44036697247706424, + "language": "python", + "language_confidence": 1.0, + "original": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n", + "original_tokens": 218, + "preserved_imports": 1, + "preserved_signatures": 2, + "symbol_scores": { + "configure": 0.5, + "teardown": 0.5 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.320627+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/e8ce85f0703b083b.json b/tests/parity/fixtures/code_aware_compressor/e8ce85f0703b083b.json new file mode 100644 index 000000000..8d0d84576 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/e8ce85f0703b083b.json @@ -0,0 +1,42 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n", + "input_sha256": "e8ce85f0703b083bca8130c3b32c97eeb453dfc6dfcde0988cbaf8d9fc75f354", + "output": { + "cache_key": null, + "compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))", + "compressed_bodies": 0, + "compressed_tokens": 145, + "compression_ratio": 0.7073170731707317, + "language": "python", + "language_confidence": 1.0, + "original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n", + "original_tokens": 205, + "preserved_imports": 3, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 1.0, + "process": 0.5, + "reset": 0.0, + "standalone": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.318760+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/f8a7f62203b3d8ba.json b/tests/parity/fixtures/code_aware_compressor/f8a7f62203b3d8ba.json new file mode 100644 index 000000000..3f5dfab3b --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/f8a7f62203b3d8ba.json @@ -0,0 +1,42 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec) -> Vec {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n", + "input_sha256": "f8a7f62203b3d8ba9970fff408a9bfd628d0dbf0b67678f89b0ffb60bbc23d5c", + "output": { + "cache_key": null, + "compressed": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec) -> Vec {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n // [1 lines omitted; calls: Processor, new, process]\n}\n\n#[derive(Debug)]", + "compressed_bodies": 0, + "compressed_tokens": 181, + "compression_ratio": 1.0168539325842696, + "language": "rust", + "language_confidence": 1.0, + "original": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec) -> Vec {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n", + "original_tokens": 178, + "preserved_imports": 1, + "preserved_signatures": 1, + "symbol_scores": { + "Processor": 1.0, + "main": 0.0, + "new": 0.333, + "process": 0.0 + }, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.330326+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/fixtures/code_aware_compressor/f9a7c264592ff8c4.json b/tests/parity/fixtures/code_aware_compressor/f9a7c264592ff8c4.json new file mode 100644 index 000000000..4e9c79511 --- /dev/null +++ b/tests/parity/fixtures/code_aware_compressor/f9a7c264592ff8c4.json @@ -0,0 +1,37 @@ +{ + "config": { + "ccr_ttl": 300, + "compress_comments": true, + "docstring_mode": "first_line", + "enable_ccr": false, + "fallback_to_kompress": false, + "language_hint": null, + "max_body_lines": 5, + "min_tokens_for_compression": 100, + "preserve_decorators": true, + "preserve_imports": true, + "preserve_signatures": true, + "preserve_type_annotations": true, + "semantic_analysis": true, + "target_compression_rate": 0.2 + }, + "input": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe = T | null;\n\nexport function lookup(users: User[], id: number): Maybe {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n", + "input_sha256": "f9a7c264592ff8c4f726600ef7a89beb9ef1a625cad83a6d1700f09466778889", + "output": { + "cache_key": null, + "compressed": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe = T | null;\n\nexport function lookup(users: User[], id: number): Maybe {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n", + "compressed_bodies": 0, + "compressed_tokens": 129, + "compression_ratio": 1.0, + "language": "typescript", + "language_confidence": 1.0, + "original": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe = T | null;\n\nexport function lookup(users: User[], id: number): Maybe {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n", + "original_tokens": 129, + "preserved_imports": 0, + "preserved_signatures": 0, + "symbol_scores": {}, + "syntax_valid": true + }, + "recorded_at": "2026-06-19T00:15:16.325460+00:00", + "transform": "code_aware_compressor" +} diff --git a/tests/parity/recorder.py b/tests/parity/recorder.py index 654ef3858..c27a105e7 100644 --- a/tests/parity/recorder.py +++ b/tests/parity/recorder.py @@ -294,6 +294,20 @@ def record_all(root: Path | None = None) -> dict[str, str]: except Exception as e: statuses["kompress"] = f"blocked:{e.__class__.__name__}:{e}" + # --- code_aware_compressor --------------------------------------------- + # AST code compressor. The workload driver installs an individual-grammar + # parser patch (the installed tree-sitter-language-pack is an + # API-incompatible native binding) and constructs it with enable_ccr=False + # + fallback_to_kompress=False so output is deterministic and + # store/model-independent. + try: + from headroom.transforms.code_compressor import CodeAwareCompressor + + _wrap_method(CodeAwareCompressor, "compress", "code_aware_compressor", root=root) + statuses["code_aware_compressor"] = "patched" + except Exception as e: + statuses["code_aware_compressor"] = f"blocked:{e.__class__.__name__}:{e}" + return statuses @@ -765,6 +779,7 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]: "ccr": 0, "content_detector": 0, "kompress": 0, + "code_aware_compressor": 0, } # log_compressor @@ -871,9 +886,439 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]: except Exception as e: LOG.warning("kompress workload failed: %s", e) + # code_aware_compressor — AST code compression over all 8 languages. + # Installs the individual-grammar parser patch first (see + # install_individual_grammar_parsers); enable_ccr=False + + # fallback_to_kompress=False keep output deterministic. Soft-fails when + # the per-language grammar wheels aren't installed. + try: + from headroom.transforms.code_compressor import ( + CodeAwareCompressor, + CodeCompressorConfig, + ) + + install_individual_grammar_parsers() + cac = CodeAwareCompressor( + CodeCompressorConfig(enable_ccr=False, fallback_to_kompress=False) + ) + for s in _varied_code_inputs(): + cac.compress(s) + counts["code_aware_compressor"] += 1 + except Exception as e: + LOG.warning("code_aware_compressor workload failed: %s", e) + return counts +def install_individual_grammar_parsers() -> None: + """Repoint `code_compressor._get_parser` at the individual + `tree-sitter-` grammar wheels (pinned to match the Rust crates). + + The installed `tree-sitter-language-pack` is an alef-generated native + binding whose `get_language()` returns a non-`tree_sitter.Language` and + whose `parse()` wants `str` — API-incompatible with `code_compressor.py` + (which builds a stock `tree_sitter.Parser` from a `tree_sitter.Language`). + To record fixtures against grammars the Rust port can match byte-for-byte, + we swap in stock `tree_sitter.Parser`s bound to the per-language grammars. + Raises ImportError when the grammar wheels / core binding aren't present + (the workload driver soft-fails, like kompress without its model). + """ + import tree_sitter_c + import tree_sitter_cpp + import tree_sitter_go + import tree_sitter_java + import tree_sitter_javascript + import tree_sitter_python + import tree_sitter_rust + import tree_sitter_typescript + from tree_sitter import Language, Parser + + from headroom.transforms import code_compressor as cc + + langs = { + "python": Language(tree_sitter_python.language()), + "javascript": Language(tree_sitter_javascript.language()), + "typescript": Language(tree_sitter_typescript.language_typescript()), + "go": Language(tree_sitter_go.language()), + "rust": Language(tree_sitter_rust.language()), + "java": Language(tree_sitter_java.language()), + "c": Language(tree_sitter_c.language()), + "cpp": Language(tree_sitter_cpp.language()), + } + cache: dict[str, Any] = {} + + def _get_parser(language: str) -> Any: + if language not in cache: + try: + cache[language] = Parser(langs[language]) + except TypeError: # older binding: assign .language + p = Parser() + p.language = langs[language] + cache[language] = p + return cache[language] + + cc._get_parser = _get_parser # type: ignore[assignment] # noqa: SLF001 + cc._check_tree_sitter_available = lambda: True # type: ignore[assignment] # noqa: SLF001 + + +def _varied_code_inputs() -> list[str]: + """≥20 varied source-code inputs spanning all 8 supported languages. + + Exercises: imports, long-bodied functions (body truncation), classes with + multiple methods, decorators, type definitions, top-level code, Python + docstring first-line reconstruction, short passthrough (<100 tokens), and + an UNKNOWN input (plain prose → passthrough with fallback disabled). All + inputs are ASCII (the non-ASCII byte/char slice ambiguity is out of parity + scope; see code_compressor.rs module docs).""" + python_basic = ( + "import os\n" + "import sys\n" + "from typing import List, Optional\n\n" + "GLOBAL_CONST = 42\n\n" + "@dataclass\n" + "class Processor:\n" + ' """Process a stream of items efficiently."""\n' + " name: str\n" + " count: int = 0\n\n" + " def process(self, items: List[str]) -> List[str]:\n" + ' """Process a list of items and return cleaned results."""\n' + " results = []\n" + " for item in items:\n" + " if not item:\n" + " continue\n" + " processed = item.strip().lower()\n" + " results.append(processed)\n" + " self.count += 1\n" + " return results\n\n" + " def reset(self):\n" + " self.count = 0\n\n\n" + "def standalone(x: int, y: int) -> int:\n" + " total = 0\n" + " for i in range(x):\n" + " for j in range(y):\n" + " total += i * j\n" + " return total\n\n\n" + 'if __name__ == "__main__":\n' + ' p = Processor("main")\n' + ' print(p.process(["a", "b"]))\n' + ) + python_nodoc = ( + "from collections import defaultdict\n\n\n" + "def build_index(records):\n" + " index = defaultdict(list)\n" + " for rec in records:\n" + ' key = rec.get("id")\n' + " if key is None:\n" + " continue\n" + " index[key].append(rec)\n" + " if len(index[key]) > 100:\n" + " index[key] = index[key][:100]\n" + " return index\n\n\n" + "def merge(a, b):\n" + " out = dict(a)\n" + " for k, v in b.items():\n" + " out[k] = v\n" + " return out\n" + ) + python_multiline_ds = ( + "import logging\n\n\n" + "def configure(level, handlers, fmt, propagate, capture_warnings):\n" + ' """Configure logging for the whole application.\n\n' + " Sets up the root logger with the given level and handler list,\n" + " applies the format string to every handler, and toggles warning\n" + " capture so library warnings are routed through the logger too.\n" + ' """\n' + " logger = logging.getLogger()\n" + " logger.setLevel(level)\n" + " formatter = logging.Formatter(fmt)\n" + " for handler in handlers:\n" + " handler.setFormatter(formatter)\n" + " logger.addHandler(handler)\n" + " logger.propagate = propagate\n" + " logging.captureWarnings(capture_warnings)\n" + ' logger.info("logging configured at level %s", level)\n' + " return logger\n\n\n" + "def teardown(logger):\n" + " for handler in list(logger.handlers):\n" + " handler.flush()\n" + " handler.close()\n" + " logger.removeHandler(handler)\n" + " return logger\n" + ) + javascript_basic = ( + "import { foo } from './foo';\n" + "const bar = require('bar');\n\n" + "const CONST = 99;\n\n" + "export function processData(items) {\n" + " const results = [];\n" + " for (const item of items) {\n" + " if (!item) {\n" + " continue;\n" + " }\n" + " const clean = item.trim().toLowerCase();\n" + " results.push(clean);\n" + " }\n" + " return results;\n" + "}\n\n" + "class Widget {\n" + " constructor(name) {\n" + " this.name = name;\n" + " this.count = 0;\n" + " }\n\n" + " render(ctx) {\n" + " ctx.clear();\n" + " ctx.draw(this.name);\n" + " this.count += 1;\n" + " return ctx;\n" + " }\n" + "}\n\n" + "module.exports = { processData, Widget };\n" + ) + typescript_basic = ( + "import { Foo } from './foo';\n\n" + "interface User {\n" + " id: number;\n" + " name: string;\n" + " email?: string;\n" + "}\n\n" + "type Maybe = T | null;\n\n" + "export function lookup(users: User[], id: number): Maybe {\n" + " for (const user of users) {\n" + " if (user.id === id) {\n" + " return user;\n" + " }\n" + " }\n" + " return null;\n" + "}\n\n" + "class Repository {\n" + " private items: T[] = [];\n\n" + " add(item: T): void {\n" + " this.items.push(item);\n" + " if (this.items.length > 1000) {\n" + " this.items.shift();\n" + " }\n" + " }\n" + "}\n" + ) + go_basic = ( + "package main\n\n" + "import (\n" + '\t"fmt"\n' + '\t"strings"\n' + ")\n\n" + "type Processor struct {\n" + "\tName string\n" + "\tCount int\n" + "}\n\n" + "func (p *Processor) Process(items []string) []string {\n" + "\tresults := make([]string, 0, len(items))\n" + "\tfor _, item := range items {\n" + '\t\tif item == "" {\n' + "\t\t\tcontinue\n" + "\t\t}\n" + "\t\tclean := strings.ToLower(strings.TrimSpace(item))\n" + "\t\tresults = append(results, clean)\n" + "\t\tp.Count++\n" + "\t}\n" + "\treturn results\n" + "}\n\n" + "func main() {\n" + '\tp := &Processor{Name: "main"}\n' + '\tfmt.Println(p.Process([]string{"a", "b"}))\n' + "}\n" + ) + rust_basic = ( + "use std::collections::HashMap;\n\n" + "pub struct Processor {\n" + " name: String,\n" + " count: u32,\n" + "}\n\n" + "impl Processor {\n" + " pub fn new(name: String) -> Self {\n" + " Self { name, count: 0 }\n" + " }\n\n" + " pub fn process(&mut self, items: Vec) -> Vec {\n" + " let mut results = Vec::new();\n" + " for item in items {\n" + " if item.is_empty() {\n" + " continue;\n" + " }\n" + " let clean = item.trim().to_lowercase();\n" + " results.push(clean);\n" + " self.count += 1;\n" + " }\n" + " results\n" + " }\n" + "}\n\n" + "#[derive(Debug)]\n" + "enum Color {\n" + " Red,\n" + " Green,\n" + " Blue,\n" + "}\n\n" + "fn main() {\n" + ' let mut p = Processor::new("main".to_string());\n' + ' println!("{:?}", p.process(vec![]));\n' + "}\n" + ) + java_basic = ( + "package com.example;\n\n" + "import java.util.List;\n" + "import java.util.ArrayList;\n\n" + "public class Processor {\n" + " private String name;\n" + " private int count;\n\n" + " public Processor(String name) {\n" + " this.name = name;\n" + " this.count = 0;\n" + " }\n\n" + " @Override\n" + " public String toString() {\n" + ' return "Processor(" + name + ")";\n' + " }\n\n" + " public List process(List items) {\n" + " List results = new ArrayList<>();\n" + " for (String item : items) {\n" + " if (item == null || item.isEmpty()) {\n" + " continue;\n" + " }\n" + " String clean = item.trim().toLowerCase();\n" + " results.add(clean);\n" + " this.count += 1;\n" + " }\n" + " return results;\n" + " }\n" + "}\n" + ) + c_basic = ( + "#include \n" + "#include \n" + "#include \n\n" + "typedef struct {\n" + " char name[64];\n" + " int count;\n" + "} Processor;\n\n" + "int process(Processor *p, const char **items, int n) {\n" + " int kept = 0;\n" + " for (int i = 0; i < n; i++) {\n" + " if (items[i] == NULL || strlen(items[i]) == 0) {\n" + " continue;\n" + " }\n" + " kept++;\n" + " p->count++;\n" + " }\n" + " return kept;\n" + "}\n\n" + "int main(void) {\n" + ' Processor p = {"main", 0};\n' + ' const char *items[] = {"a", "b"};\n' + ' printf("%d\\n", process(&p, items, 2));\n' + " return 0;\n" + "}\n" + ) + cpp_basic = ( + "#include \n" + "#include \n" + "#include \n\n" + "namespace app {\n\n" + "class Processor {\n" + "public:\n" + " Processor(const std::string &name) : name_(name), count_(0) {}\n\n" + " std::vector process(const std::vector &items) {\n" + " std::vector results;\n" + " for (const auto &item : items) {\n" + " if (item.empty()) {\n" + " continue;\n" + " }\n" + " results.push_back(item);\n" + " count_++;\n" + " }\n" + " return results;\n" + " }\n\n" + "private:\n" + " std::string name_;\n" + " int count_;\n" + "};\n\n" + "} // namespace app\n\n" + "int main() {\n" + ' app::Processor p("main");\n' + " return 0;\n" + "}\n" + ) + # Short (<100 char/4 tokens) → passthrough, per a couple languages. + short_py = "def f(x):\n return x + 1\n" + short_js = "const a = 1;\nfunction g() { return a; }\n" + # UNKNOWN → passthrough (fallback_to_kompress=False in the recorder). + unknown_prose = ( + "This is just a paragraph of plain English prose that contains no " + "recognizable source code constructs at all, so the language detector " + "should classify it as unknown and the compressor should pass it " + "through unchanged without attempting any AST based compression here.\n" + ) + # A longer multi-function Python file to exercise budget allocation. + python_orchestrator = ( + "import json\n\n\n" + "def load(path):\n" + " with open(path) as fh:\n" + " data = json.load(fh)\n" + " cleaned = {}\n" + " for key, value in data.items():\n" + " if value is None:\n" + " continue\n" + " cleaned[key] = value\n" + " return cleaned\n\n\n" + "def transform(records, factor):\n" + " out = []\n" + " for r in records:\n" + " scaled = r * factor\n" + " if scaled > 1000:\n" + " scaled = 1000\n" + " out.append(scaled)\n" + " return out\n\n\n" + "def run(path, factor):\n" + " data = load(path)\n" + " values = list(data.values())\n" + " result = transform(values, factor)\n" + " return sum(result)\n" + ) + + return [ + python_basic, + python_nodoc, + python_multiline_ds, + python_orchestrator, + javascript_basic, + typescript_basic, + go_basic, + rust_basic, + java_basic, + c_basic, + cpp_basic, + short_py, + short_js, + unknown_prose, + # Padded variants for >= 20 unique inputs. A trailing comment keeps + # each input unique (distinct fixture hash) while preserving the + # structural paths exercised above. + f"{python_basic}\n# variant 1", + f"{javascript_basic}\n// variant 1", + f"{go_basic}\n// variant 1", + f"{rust_basic}\n// variant 1", + f"{java_basic}\n// variant 1", + f"{c_basic}\n// variant 1", + f"{cpp_basic}\n// variant 1", + f"{typescript_basic}\n// variant 1", + f"{python_orchestrator}\n# variant 1", + ] + + +def _docstring_mode_inputs() -> list[str]: + """Docstring-bearing Python samples, for exercising the non-default + `DocstringMode` branches (FULL keeps the whole docstring, REMOVE drops it). + The default `_varied_code_inputs` only exercises FIRST_LINE.""" + return [s for s in _varied_code_inputs() if '"""' in s][:2] + + def _varied_kompress_inputs() -> list[str]: """≥20 varied prose/log/mixed inputs for the Kompress ML compressor.