mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-10 14:27:00 -04:00
feat(rust): port CodeCompressor AST compressor to Rust (parity-only) (#1154)
Adds crates/headroom-core/src/transforms/code_compressor.rs (1,882 lines): the AST-aware CodeCompressor ported to Rust on tree-sitter, with grammars for Python, JavaScript, TypeScript, Go, Rust, Java, C and C++. Parity-only, like #1153. Nothing calls it: the only references outside the module are the pub mod / pub use declarations in transforms/mod.rs, and live_zone.rs still routes SourceCode to a no-op. The pyo3 bridge is untouched and no Python source changes, so the engine is unreachable from the shipped package. #1155 wires it into live-zone dispatch. Every grammar is pinned with '=' to the exact version of the corresponding Python tree-sitter-<lang> PyPI wheel. Same version on crates.io and PyPI means the same grammar.js, hence the same generated parser.c, hence node-for-node identical ASTs — the precondition for byte-parity. A canary over 9 samples x 8 languages confirmed identical node-type and line-span trees at these pins; bumping any pin requires re-running it and re-recording the fixtures. Ships 30 recorded parity fixtures, a CodeCompressorComparator in headroom-parity, and scripts/record_code_compressor_fixtures.py. Verified byte-identical to the recorded Python output: [code_aware_compressor] total=30 matched=30 skipped=0 diffed=0 Full harness on the merge result: 227 fixtures, 182 matched, 45 skipped (cache_aligner + ccr stubs), 0 diffed, exit 0 — with kompress at 21/21 under ONNX Runtime 1.24.4 (see #2591). Also verified cargo check -p headroom-core --no-default-features passes, so the static-musl path stays intact.
This commit is contained in:
Ruben A.
GitHub
parent
83e27e5036
commit
e530de5ad2
39 changed files with 3991 additions and 0 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -39,6 +39,7 @@ scripts/*
|
|||
!scripts/replay_codex_ws_load.py
|
||||
!scripts/export_kompress_v2_onnx.py
|
||||
!scripts/record_kompress_fixtures.py
|
||||
!scripts/record_code_compressor_fixtures.py
|
||||
|
||||
# Rust / Cargo build artifacts
|
||||
/target/
|
||||
|
|
|
|||
115
Cargo.lock
generated
115
Cargo.lock
generated
|
|
@ -1899,6 +1899,15 @@ dependencies = [
|
|||
"tokenizers",
|
||||
"toml",
|
||||
"tracing",
|
||||
"tree-sitter",
|
||||
"tree-sitter-c",
|
||||
"tree-sitter-cpp",
|
||||
"tree-sitter-go",
|
||||
"tree-sitter-java",
|
||||
"tree-sitter-javascript",
|
||||
"tree-sitter-python",
|
||||
"tree-sitter-rust",
|
||||
"tree-sitter-typescript",
|
||||
"unidiff",
|
||||
]
|
||||
|
||||
|
|
@ -4108,6 +4117,12 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
|
|
@ -4614,6 +4629,106 @@ dependencies = [
|
|||
"tracing-serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter"
|
||||
version = "0.25.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5168a515fe492af54c5cc8800ff8c840be09fa5168de45838afaecd3e008bce4"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"serde_json",
|
||||
"streaming-iterator",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-c"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9b2eb57a55fed6b00812912e730b7a275cf4fe98bfd6a5d76263d4438371728"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-cpp"
|
||||
version = "0.23.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-go"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-java"
|
||||
version = "0.23.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-javascript"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-python"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-rust"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-typescript"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "try-lock"
|
||||
version = "0.2.5"
|
||||
|
|
|
|||
|
|
@ -137,6 +137,25 @@ redis = { version = "0.27", optional = true, default-features = false }
|
|||
# classifier live with the other Phase B/F policy primitives without
|
||||
# cycling through the proxy crate. Tiny crate (no I/O, just types).
|
||||
http = "1"
|
||||
# tree-sitter + per-language grammars for the CodeCompressor AST port.
|
||||
# Versions are pinned to EXACTLY match the Python reference grammars
|
||||
# (`tree-sitter-<lang>` PyPI wheels) so the Rust and Python parsers emit
|
||||
# node-for-node identical ASTs — the precondition for byte-parity. Same
|
||||
# version number on crates.io + PyPI means the same `grammar.js` source,
|
||||
# hence the same generated `parser.c`. The grammar-parity canary (9
|
||||
# samples × 8 languages) confirmed 100% identical node-type + line-span
|
||||
# trees at these exact pins. Bumping any pin requires re-running the
|
||||
# canary and re-recording the code_aware_compressor fixtures.
|
||||
tree-sitter = "=0.25.2"
|
||||
tree-sitter-python = "=0.25.0"
|
||||
tree-sitter-javascript = "=0.25.0"
|
||||
tree-sitter-typescript = "=0.23.2"
|
||||
tree-sitter-go = "=0.25.0"
|
||||
tree-sitter-rust = "=0.24.2"
|
||||
tree-sitter-java = "=0.23.5"
|
||||
tree-sitter-c = "=0.24.2"
|
||||
tree-sitter-cpp = "=0.23.4"
|
||||
|
||||
# Load ONNX Runtime dynamically on every platform. The alternative,
|
||||
# `ort-download-binaries-*`, statically links Microsoft's prebuilt ORT:
|
||||
# on Windows it emits DirectML link libs (`DXCORE`, `DXGI`, `D3D12`,
|
||||
|
|
|
|||
1882
crates/headroom-core/src/transforms/code_compressor.rs
Normal file
1882
crates/headroom-core/src/transforms/code_compressor.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
pub mod adaptive_sizer;
|
||||
pub mod anchor_selector;
|
||||
pub mod code_compressor;
|
||||
pub mod content_detector;
|
||||
pub mod detection;
|
||||
pub mod diff_compressor;
|
||||
|
|
@ -35,6 +36,10 @@ pub mod tag_protector;
|
|||
pub mod text_crusher;
|
||||
pub mod unidiff_detector;
|
||||
|
||||
pub use code_compressor::{
|
||||
detect_language, CodeAwareCompressor, CodeCompressionResult, CodeCompressorConfig,
|
||||
CodeLanguage, DocstringMode,
|
||||
};
|
||||
pub use content_detector::{
|
||||
detect_content_type, is_json_array_of_dicts, ContentType, DetectionResult,
|
||||
};
|
||||
|
|
|
|||
150
crates/headroom-core/tests/code_compressor_parity.rs
Normal file
150
crates/headroom-core/tests/code_compressor_parity.rs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
//! Byte-parity integration test for the CodeCompressor Rust port.
|
||||
//!
|
||||
//! Runs the production [`CodeAwareCompressor`] against the fixtures recorded
|
||||
//! from the Python reference (`tests/parity/fixtures/code_aware_compressor/`)
|
||||
//! and asserts the serialized result matches field-for-field.
|
||||
//!
|
||||
//! Unlike the Kompress test, this needs no model/network: the per-language
|
||||
//! tree-sitter grammars are compiled into the crate. Grammar-version parity
|
||||
//! is guaranteed by the exact Cargo pins matching the Python wheels the
|
||||
//! fixtures were recorded against (see `Cargo.toml`).
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use headroom_core::transforms::code_compressor::{
|
||||
CodeAwareCompressor, CodeCompressorConfig, DocstringMode,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
fn config_from_fixture(config: &Value) -> CodeCompressorConfig {
|
||||
let d = CodeCompressorConfig::default();
|
||||
CodeCompressorConfig {
|
||||
preserve_imports: config
|
||||
.get("preserve_imports")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.preserve_imports),
|
||||
preserve_signatures: config
|
||||
.get("preserve_signatures")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.preserve_signatures),
|
||||
preserve_type_annotations: config
|
||||
.get("preserve_type_annotations")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.preserve_type_annotations),
|
||||
preserve_decorators: config
|
||||
.get("preserve_decorators")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.preserve_decorators),
|
||||
docstring_mode: config
|
||||
.get("docstring_mode")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(DocstringMode::from_value)
|
||||
.unwrap_or(d.docstring_mode),
|
||||
target_compression_rate: config
|
||||
.get("target_compression_rate")
|
||||
.and_then(|v| v.as_f64())
|
||||
.unwrap_or(d.target_compression_rate),
|
||||
max_body_lines: config
|
||||
.get("max_body_lines")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(d.max_body_lines),
|
||||
compress_comments: config
|
||||
.get("compress_comments")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.compress_comments),
|
||||
min_tokens_for_compression: config
|
||||
.get("min_tokens_for_compression")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(d.min_tokens_for_compression),
|
||||
language_hint: config
|
||||
.get("language_hint")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
fallback_to_kompress: config
|
||||
.get("fallback_to_kompress")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.fallback_to_kompress),
|
||||
semantic_analysis: config
|
||||
.get("semantic_analysis")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.semantic_analysis),
|
||||
enable_ccr: config
|
||||
.get("enable_ccr")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(d.enable_ccr),
|
||||
ccr_ttl: config
|
||||
.get("ccr_ttl")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(d.ccr_ttl),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_compressor_matches_python_fixtures_byte_for_byte() {
|
||||
let fixtures_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../tests/parity/fixtures/code_aware_compressor");
|
||||
assert!(
|
||||
fixtures_dir.exists(),
|
||||
"fixtures dir {} missing — run scripts/record_code_compressor_fixtures.py",
|
||||
fixtures_dir.display()
|
||||
);
|
||||
|
||||
let mut paths: Vec<PathBuf> = fs::read_dir(&fixtures_dir)
|
||||
.expect("read fixtures dir")
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.extension().map(|x| x == "json").unwrap_or(false))
|
||||
.collect();
|
||||
paths.sort();
|
||||
|
||||
let mut checked = 0usize;
|
||||
let mut nontrivial = 0usize;
|
||||
for path in &paths {
|
||||
let fx: Value = serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap();
|
||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let content = fx["input"].as_str().expect("fixture.input string");
|
||||
let expected = &fx["output"];
|
||||
|
||||
let cfg = config_from_fixture(&fx["config"]);
|
||||
let result = CodeAwareCompressor::new(cfg).compress(content);
|
||||
|
||||
let mut symbol_scores = serde_json::Map::new();
|
||||
for (k, v) in &result.symbol_scores {
|
||||
symbol_scores.insert(k.clone(), serde_json::json!(v));
|
||||
}
|
||||
let actual = serde_json::json!({
|
||||
"cache_key": result.cache_key,
|
||||
"compressed": result.compressed,
|
||||
"compressed_bodies": result.compressed_bodies,
|
||||
"compressed_tokens": result.compressed_tokens,
|
||||
"compression_ratio": result.compression_ratio,
|
||||
"language": result.language.value(),
|
||||
"language_confidence": result.language_confidence,
|
||||
"original": result.original,
|
||||
"original_tokens": result.original_tokens,
|
||||
"preserved_imports": result.preserved_imports,
|
||||
"preserved_signatures": result.preserved_signatures,
|
||||
"symbol_scores": serde_json::Value::Object(symbol_scores),
|
||||
"syntax_valid": result.syntax_valid,
|
||||
});
|
||||
// Normalize through serde (f64 round-trip), matching the harness.
|
||||
let actual: Value = serde_json::from_str(&serde_json::to_string(&actual).unwrap()).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
&actual, expected,
|
||||
"[{name}] code compressor output diverged from Python reference"
|
||||
);
|
||||
if result.compressed != content {
|
||||
nontrivial += 1;
|
||||
}
|
||||
checked += 1;
|
||||
}
|
||||
|
||||
assert!(checked >= 20, "expected >= 20 fixtures, got {checked}");
|
||||
assert!(
|
||||
nontrivial >= 10,
|
||||
"expected >= 10 non-trivial compressions, got {nontrivial} (are the fixtures all passthroughs?)"
|
||||
);
|
||||
eprintln!("code_compressor parity: {checked} fixtures matched ({nontrivial} non-trivial)");
|
||||
}
|
||||
|
|
@ -713,6 +713,124 @@ impl TransformComparator for KompressComparator {
|
|||
}
|
||||
}
|
||||
|
||||
/// Real comparator for the `code_aware_compressor` transform. Drives the
|
||||
/// Rust AST code compressor over the recorded fixture inputs and emits the
|
||||
/// same shape Python's recorder serializes for `CodeCompressionResult`
|
||||
/// (dataclass fields via `asdict`; the `@property` derivatives are not
|
||||
/// serialized). Fixtures are recorded with `enable_ccr=False` and
|
||||
/// `fallback_to_kompress=False` so the output is deterministic and
|
||||
/// store/model-independent.
|
||||
///
|
||||
/// Grammar-version parity is the precondition: the Rust `tree-sitter-<lang>`
|
||||
/// crates are pinned to the exact versions of the Python wheels the fixtures
|
||||
/// were recorded against (see `headroom-core/Cargo.toml`).
|
||||
pub struct CodeCompressorComparator;
|
||||
|
||||
impl TransformComparator for CodeCompressorComparator {
|
||||
fn name(&self) -> &str {
|
||||
"code_aware_compressor"
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
input: &serde_json::Value,
|
||||
config: &serde_json::Value,
|
||||
) -> Result<serde_json::Value> {
|
||||
use headroom_core::transforms::code_compressor::{
|
||||
CodeAwareCompressor, CodeCompressorConfig, DocstringMode,
|
||||
};
|
||||
|
||||
let content = input
|
||||
.as_str()
|
||||
.context("code_aware_compressor fixture input must be a JSON string")?;
|
||||
|
||||
let defaults = CodeCompressorConfig::default();
|
||||
let cfg = CodeCompressorConfig {
|
||||
preserve_imports: config
|
||||
.get("preserve_imports")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.preserve_imports),
|
||||
preserve_signatures: config
|
||||
.get("preserve_signatures")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.preserve_signatures),
|
||||
preserve_type_annotations: config
|
||||
.get("preserve_type_annotations")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.preserve_type_annotations),
|
||||
preserve_decorators: config
|
||||
.get("preserve_decorators")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.preserve_decorators),
|
||||
docstring_mode: config
|
||||
.get("docstring_mode")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(DocstringMode::from_value)
|
||||
.unwrap_or(defaults.docstring_mode),
|
||||
target_compression_rate: config
|
||||
.get("target_compression_rate")
|
||||
.and_then(|v| v.as_f64())
|
||||
.unwrap_or(defaults.target_compression_rate),
|
||||
max_body_lines: config
|
||||
.get("max_body_lines")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(defaults.max_body_lines),
|
||||
compress_comments: config
|
||||
.get("compress_comments")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.compress_comments),
|
||||
min_tokens_for_compression: config
|
||||
.get("min_tokens_for_compression")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(defaults.min_tokens_for_compression),
|
||||
language_hint: config
|
||||
.get("language_hint")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
fallback_to_kompress: config
|
||||
.get("fallback_to_kompress")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.fallback_to_kompress),
|
||||
semantic_analysis: config
|
||||
.get("semantic_analysis")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.semantic_analysis),
|
||||
enable_ccr: config
|
||||
.get("enable_ccr")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(defaults.enable_ccr),
|
||||
ccr_ttl: config
|
||||
.get("ccr_ttl")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(defaults.ccr_ttl),
|
||||
};
|
||||
|
||||
let compressor = CodeAwareCompressor::new(cfg);
|
||||
let result = compressor.compress(content);
|
||||
|
||||
let mut symbol_scores = serde_json::Map::new();
|
||||
for (name, score) in &result.symbol_scores {
|
||||
symbol_scores.insert(name.clone(), serde_json::json!(score));
|
||||
}
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"cache_key": result.cache_key,
|
||||
"compressed": result.compressed,
|
||||
"compressed_bodies": result.compressed_bodies,
|
||||
"compressed_tokens": result.compressed_tokens,
|
||||
"compression_ratio": result.compression_ratio,
|
||||
"language": result.language.value(),
|
||||
"language_confidence": result.language_confidence,
|
||||
"original": result.original,
|
||||
"original_tokens": result.original_tokens,
|
||||
"preserved_imports": result.preserved_imports,
|
||||
"preserved_signatures": result.preserved_signatures,
|
||||
"symbol_scores": serde_json::Value::Object(symbol_scores),
|
||||
"syntax_valid": result.syntax_valid,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// Every built-in comparator, in a stable order.
|
||||
pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {
|
||||
vec![
|
||||
|
|
@ -725,6 +843,7 @@ pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {
|
|||
Box::new(ContentDetectorComparator),
|
||||
Box::new(TextCrusherComparator),
|
||||
Box::new(KompressComparator::new()),
|
||||
Box::new(CodeCompressorComparator),
|
||||
]
|
||||
}
|
||||
|
||||
|
|
|
|||
83
scripts/record_code_compressor_fixtures.py
Normal file
83
scripts/record_code_compressor_fixtures.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Record standard parity fixtures for the CodeAwareCompressor only.
|
||||
|
||||
Installs the individual-grammar parser patch, then drives the Python
|
||||
`CodeAwareCompressor` (enable_ccr=False, fallback_to_kompress=False) over
|
||||
`_varied_code_inputs()` while `record_all()` has the `compress` method
|
||||
patched, so only `tests/parity/fixtures/code_aware_compressor/` is
|
||||
(re)written — no churn to other transforms' fixtures.
|
||||
|
||||
The grammar wheels must be installed at the versions the Rust crates pin
|
||||
(same version number on PyPI + crates.io = same grammar source = identical
|
||||
ASTs; verified by the grammar-parity canary):
|
||||
|
||||
pip install tree-sitter==0.25.2 \\
|
||||
tree-sitter-python==0.25.0 tree-sitter-javascript==0.25.0 \\
|
||||
tree-sitter-typescript==0.23.2 tree-sitter-go==0.25.0 \\
|
||||
tree-sitter-rust==0.24.2 tree-sitter-java==0.23.5 \\
|
||||
tree-sitter-c==0.24.2 tree-sitter-cpp==0.23.4
|
||||
python scripts/record_code_compressor_fixtures.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
from tests.parity.recorder import (
|
||||
_docstring_mode_inputs,
|
||||
_varied_code_inputs,
|
||||
install_individual_grammar_parsers,
|
||||
record_all,
|
||||
)
|
||||
|
||||
statuses = record_all()
|
||||
if not statuses.get("code_aware_compressor", "").startswith("patched"):
|
||||
print(
|
||||
f"code_aware_compressor not patched: {statuses.get('code_aware_compressor')}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
install_individual_grammar_parsers()
|
||||
|
||||
from headroom.transforms.code_compressor import (
|
||||
CodeAwareCompressor,
|
||||
CodeCompressorConfig,
|
||||
DocstringMode,
|
||||
)
|
||||
|
||||
inputs = _varied_code_inputs()
|
||||
cac = CodeAwareCompressor(CodeCompressorConfig(enable_ccr=False, fallback_to_kompress=False))
|
||||
for s in inputs:
|
||||
cac.compress(s)
|
||||
|
||||
# Non-default docstring modes (FULL / REMOVE) over docstring-bearing
|
||||
# samples — distinct config hash → distinct fixtures.
|
||||
ds_inputs = _docstring_mode_inputs()
|
||||
extra = 0
|
||||
for mode in (DocstringMode.FULL, DocstringMode.REMOVE):
|
||||
c = CodeAwareCompressor(
|
||||
CodeCompressorConfig(enable_ccr=False, fallback_to_kompress=False, docstring_mode=mode)
|
||||
)
|
||||
for s in ds_inputs:
|
||||
c.compress(s)
|
||||
extra += 1
|
||||
|
||||
out_dir = REPO / "tests" / "parity" / "fixtures" / "code_aware_compressor"
|
||||
n = len(list(out_dir.glob("*.json")))
|
||||
print(
|
||||
f"recorded {n} code_aware_compressor fixtures "
|
||||
f"from {len(inputs)} default + {extra} docstring-mode inputs -> {out_dir}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List<String> process(List<String> items) {\n List<String> results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n",
|
||||
"input_sha256": "07dd1a61089c1e5326d1d4d5a404a5363fc3f45db1b46bd4554987916df4d575",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "package com.example;\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List<String> process(List<String> items) {\n List<String> results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 180,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "java",
|
||||
"language_confidence": 1.0,
|
||||
"original": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List<String> process(List<String> items) {\n List<String> results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n",
|
||||
"original_tokens": 180,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {
|
||||
"Processor": 0.071,
|
||||
"String": 1.0,
|
||||
"process": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.332197+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n",
|
||||
"input_sha256": "095be1adad76b588f996e2c39b843e28ef41ec58b2ca8bb963c4fe9389027413",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "#include <stdio.h>\n\n#include <stdlib.h>\n\n#include <string.h>\n\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n // [8 lines omitted]\n}\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 93,
|
||||
"compression_ratio": 0.7045454545454546,
|
||||
"language": "cpp",
|
||||
"language_confidence": 1.0,
|
||||
"original": "#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n",
|
||||
"original_tokens": 132,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 2,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.333831+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "const a = 1;\nfunction g() { return a; }\n",
|
||||
"input_sha256": "1e319c5e33af4af1aca7a0b5d4fe402d7efe7c4adfcce6353fe85dd55d26d7eb",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "const a = 1;\nfunction g() { return a; }\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 10,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "unknown",
|
||||
"language_confidence": 0.0,
|
||||
"original": "const a = 1;\nfunction g() { return a; }\n",
|
||||
"original_tokens": 10,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.336456+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "remove",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n",
|
||||
"input_sha256": "1e496ebaf89fab96ac4f3836146f87ea6830de66a27cbf4134068ad07be50b74",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 129,
|
||||
"compression_ratio": 0.6292682926829268,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n",
|
||||
"original_tokens": 205,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 1.0,
|
||||
"process": 0.5,
|
||||
"reset": 0.0,
|
||||
"standalone": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.358515+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n",
|
||||
"input_sha256": "2c239d4af41282c27453261058252f76b9f173d4b16d69fa1094010556969416",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import json\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n # [6 lines omitted]\n pass\ndef transform(records, factor):\n out = []\n # [6 lines omitted]\n pass\ndef run(path, factor):\n data = load(path)\n # [3 lines omitted; calls: load, transform]\n pass",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 74,
|
||||
"compression_ratio": 0.524822695035461,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n",
|
||||
"original_tokens": 141,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 3,
|
||||
"symbol_scores": {
|
||||
"load": 1.0,
|
||||
"run": 0.0,
|
||||
"transform": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.321701+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n\n# variant 1",
|
||||
"input_sha256": "2d214f401618d6392d0d152fc1446571e6f28045d51569b98af9bf59841eb969",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import json\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n # [6 lines omitted]\n pass\ndef transform(records, factor):\n out = []\n # [6 lines omitted]\n pass\ndef run(path, factor):\n data = load(path)\n # [3 lines omitted; calls: load, transform]\n pass\n\n# variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 77,
|
||||
"compression_ratio": 0.5347222222222222,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import json\n\n\ndef load(path):\n with open(path) as fh:\n data = json.load(fh)\n cleaned = {}\n for key, value in data.items():\n if value is None:\n continue\n cleaned[key] = value\n return cleaned\n\n\ndef transform(records, factor):\n out = []\n for r in records:\n scaled = r * factor\n if scaled > 1000:\n scaled = 1000\n out.append(scaled)\n return out\n\n\ndef run(path, factor):\n data = load(path)\n values = list(data.values())\n result = transform(values, factor)\n return sum(result)\n\n# variant 1",
|
||||
"original_tokens": 144,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 3,
|
||||
"symbol_scores": {
|
||||
"load": 1.0,
|
||||
"run": 0.0,
|
||||
"transform": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.353260+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "def f(x):\n return x + 1\n",
|
||||
"input_sha256": "36e5ff5093790e6fc3473727b0f49b944a68d1e9b0417ac97c4d0c18536d4d96",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "def f(x):\n return x + 1\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 6,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "unknown",
|
||||
"language_confidence": 0.0,
|
||||
"original": "def f(x):\n return x + 1\n",
|
||||
"original_tokens": 6,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.336336+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n\n// variant 1",
|
||||
"input_sha256": "3e706cfb03aa3c8edd80d11d4f3ce508aba524ac79f5e2f7647bc28c0ec01217",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 150,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "javascript",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n\n// variant 1",
|
||||
"original_tokens": 150,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.340479+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n\n// variant 1",
|
||||
"input_sha256": "411d5549bcd5f50dc15031bec587e643ea032e4975c90fb36d6fe7ed79195b94",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "#include <stdio.h>\n\n#include <stdlib.h>\n\n#include <string.h>\n\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n // [8 lines omitted]\n}\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 96,
|
||||
"compression_ratio": 0.7111111111111111,
|
||||
"language": "cpp",
|
||||
"language_confidence": 1.0,
|
||||
"original": "#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n\ntypedef struct {\n char name[64];\n int count;\n} Processor;\n\nint process(Processor *p, const char **items, int n) {\n int kept = 0;\n for (int i = 0; i < n; i++) {\n if (items[i] == NULL || strlen(items[i]) == 0) {\n continue;\n }\n kept++;\n p->count++;\n }\n return kept;\n}\n\nint main(void) {\n Processor p = {\"main\", 0};\n const char *items[] = {\"a\", \"b\"};\n printf(\"%d\\n\", process(&p, items, 2));\n return 0;\n}\n\n// variant 1",
|
||||
"original_tokens": 135,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 2,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.348562+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "#include <iostream>\n#include <vector>\n#include <string>\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n",
|
||||
"input_sha256": "420dd19a087526ca88d1cc71110894f73029cee57533b406ed005b382200273b",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "#include <iostream>\n\n#include <vector>\n\n#include <string>\n\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n}\n// namespace app",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 278,
|
||||
"compression_ratio": 1.759493670886076,
|
||||
"language": "cpp",
|
||||
"language_confidence": 1.0,
|
||||
"original": "#include <iostream>\n#include <vector>\n#include <string>\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n",
|
||||
"original_tokens": 158,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 0.5
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.336058+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "full",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n",
|
||||
"input_sha256": "53030c7c11ce0082b64002c5a4d5b920039825459494a5ee3dd09dd48b040b34",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 145,
|
||||
"compression_ratio": 0.7073170731707317,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n",
|
||||
"original_tokens": 205,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 1.0,
|
||||
"process": 0.5,
|
||||
"reset": 0.0,
|
||||
"standalone": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.355261+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List<String> process(List<String> items) {\n List<String> results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n\n// variant 1",
|
||||
"input_sha256": "5c057f7bfd56af058ab2f8745bf0242d97e3a458ce8efedae8263066c1c709a7",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "package com.example;\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List<String> process(List<String> items) {\n List<String> results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 183,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "java",
|
||||
"language_confidence": 1.0,
|
||||
"original": "package com.example;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\npublic class Processor {\n private String name;\n private int count;\n\n public Processor(String name) {\n this.name = name;\n this.count = 0;\n }\n\n @Override\n public String toString() {\n return \"Processor(\" + name + \")\";\n }\n\n public List<String> process(List<String> items) {\n List<String> results = new ArrayList<>();\n for (String item : items) {\n if (item == null || item.isEmpty()) {\n continue;\n }\n String clean = item.trim().toLowerCase();\n results.add(clean);\n this.count += 1;\n }\n return results;\n }\n}\n\n// variant 1",
|
||||
"original_tokens": 183,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {
|
||||
"Processor": 0.071,
|
||||
"String": 1.0,
|
||||
"process": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.346983+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "remove",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"input_sha256": "6c04d398f9c3014eca3854ea1146e9f3ce76f3611b3cabc7170347e2c550200f",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 82,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "unknown",
|
||||
"language_confidence": 0.0,
|
||||
"original": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"original_tokens": 82,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:12:54.716311+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n\n// variant 1",
|
||||
"input_sha256": "716a5a2380907e7165334f0d0ce4c4502d6756647f8cb24dacf9b4882c1f8ede",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 120,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "go",
|
||||
"language_confidence": 1.0,
|
||||
"original": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n\n// variant 1",
|
||||
"original_tokens": 120,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.342039+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "remove",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n",
|
||||
"input_sha256": "791805bff164ee9d19721a22676e34ccb73c8ad3439f93225be2946d43140138",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import logging\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n logger = logging.getLogger()\n # [9 lines omitted]\n pass\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n # [1 lines omitted]\n pass",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 82,
|
||||
"compression_ratio": 0.3761467889908257,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n",
|
||||
"original_tokens": 218,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 2,
|
||||
"symbol_scores": {
|
||||
"configure": 0.5,
|
||||
"teardown": 0.5
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.359509+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n\n# variant 1",
|
||||
"input_sha256": "7cee85d20b3604dd04b1354bc454db103725b7e76a2973b23a4b16c10800b629",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n# variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 148,
|
||||
"compression_ratio": 0.7115384615384616,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n\n# variant 1",
|
||||
"original_tokens": 208,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 1.0,
|
||||
"process": 0.5,
|
||||
"reset": 0.0,
|
||||
"standalone": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.338646+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "full",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"input_sha256": "88fe6b4b0a9908c54185177fa5adfdd8533ececd54e57ba74391fd228ee685c4",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 82,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "unknown",
|
||||
"language_confidence": 0.0,
|
||||
"original": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"original_tokens": 82,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:12:54.714085+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "from collections import defaultdict\n\n\ndef build_index(records):\n index = defaultdict(list)\n for rec in records:\n key = rec.get(\"id\")\n if key is None:\n continue\n index[key].append(rec)\n if len(index[key]) > 100:\n index[key] = index[key][:100]\n return index\n\n\ndef merge(a, b):\n out = dict(a)\n for k, v in b.items():\n out[k] = v\n return out\n",
|
||||
"input_sha256": "921876ad5dd26cf623a97a7668ec4a1026f9f8b4e2d97af154943f29482c160c",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "from collections import defaultdict\n\ndef build_index(records):\n index = defaultdict(list)\n # [8 lines omitted]\n pass\ndef merge(a, b):\n out = dict(a)\n # [3 lines omitted]\n pass",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 48,
|
||||
"compression_ratio": 0.46601941747572817,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "from collections import defaultdict\n\n\ndef build_index(records):\n index = defaultdict(list)\n for rec in records:\n key = rec.get(\"id\")\n if key is None:\n continue\n index[key].append(rec)\n if len(index[key]) > 100:\n index[key] = index[key][:100]\n return index\n\n\ndef merge(a, b):\n out = dict(a)\n for k, v in b.items():\n out[k] = v\n return out\n",
|
||||
"original_tokens": 103,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 2,
|
||||
"symbol_scores": {
|
||||
"build_index": 0.5,
|
||||
"merge": 0.5
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.319542+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n",
|
||||
"input_sha256": "b16202e9b81a5bba3ca5c9ff0ecf632870f77938a98bb41c1f0026e57b403753",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 147,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "javascript",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import { foo } from './foo';\nconst bar = require('bar');\n\nconst CONST = 99;\n\nexport function processData(items) {\n const results = [];\n for (const item of items) {\n if (!item) {\n continue;\n }\n const clean = item.trim().toLowerCase();\n results.push(clean);\n }\n return results;\n}\n\nclass Widget {\n constructor(name) {\n this.name = name;\n this.count = 0;\n }\n\n render(ctx) {\n ctx.clear();\n ctx.draw(this.name);\n this.count += 1;\n return ctx;\n }\n}\n\nmodule.exports = { processData, Widget };\n",
|
||||
"original_tokens": 147,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.323440+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n",
|
||||
"input_sha256": "b753b4725da91f71b39e71568539f9d307bf18184cc91ce2558d08865ac710f5",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 117,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "go",
|
||||
"language_confidence": 1.0,
|
||||
"original": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype Processor struct {\n\tName string\n\tCount int\n}\n\nfunc (p *Processor) Process(items []string) []string {\n\tresults := make([]string, 0, len(items))\n\tfor _, item := range items {\n\t\tif item == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tclean := strings.ToLower(strings.TrimSpace(item))\n\t\tresults = append(results, clean)\n\t\tp.Count++\n\t}\n\treturn results\n}\n\nfunc main() {\n\tp := &Processor{Name: \"main\"}\n\tfmt.Println(p.Process([]string{\"a\", \"b\"}))\n}\n",
|
||||
"original_tokens": 117,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.327065+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n\n// variant 1",
|
||||
"input_sha256": "b8fa00d165cd059a0549473920e4d61bcbde3980257d328f2cbc76fc3a3db787",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n // [1 lines omitted; calls: Processor, new, process]\n}\n\n#[derive(Debug)]\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 185,
|
||||
"compression_ratio": 1.022099447513812,
|
||||
"language": "rust",
|
||||
"language_confidence": 1.0,
|
||||
"original": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n\n// variant 1",
|
||||
"original_tokens": 181,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 1.0,
|
||||
"main": 0.0,
|
||||
"new": 0.333,
|
||||
"process": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.345389+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "#include <iostream>\n#include <vector>\n#include <string>\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\n// variant 1",
|
||||
"input_sha256": "ba9c0fda1beda6ec7b8447e7aa39a743f073cf02e914f52e99aaa6ebda3c4dc1",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "#include <iostream>\n\n#include <vector>\n\n#include <string>\n\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n}\n// namespace app\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 281,
|
||||
"compression_ratio": 1.7453416149068324,
|
||||
"language": "cpp",
|
||||
"language_confidence": 1.0,
|
||||
"original": "#include <iostream>\n#include <vector>\n#include <string>\n\nnamespace app {\n\nclass Processor {\npublic:\n Processor(const std::string &name) : name_(name), count_(0) {}\n\n std::vector<std::string> process(const std::vector<std::string> &items) {\n std::vector<std::string> results;\n for (const auto &item : items) {\n if (item.empty()) {\n continue;\n }\n results.push_back(item);\n count_++;\n }\n return results;\n }\n\nprivate:\n std::string name_;\n int count_;\n};\n\n} // namespace app\n\nint main() {\n app::Processor p(\"main\");\n return 0;\n}\n\n// variant 1",
|
||||
"original_tokens": 161,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 0.5
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.350553+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe<T> = T | null;\n\nexport function lookup(users: User[], id: number): Maybe<User> {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository<T> {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n\n// variant 1",
|
||||
"input_sha256": "bd14bdd440e5a1ab8bef7b20c809ac1617a9be9f2ba8b700e7f518bfb474e7da",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe<T> = T | null;\n\nexport function lookup(users: User[], id: number): Maybe<User> {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository<T> {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n\n// variant 1",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 132,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "typescript",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe<T> = T | null;\n\nexport function lookup(users: User[], id: number): Maybe<User> {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository<T> {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n\n// variant 1",
|
||||
"original_tokens": 132,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.352262+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"input_sha256": "bfcb158739b34b4381a4c23e9ff84bbe04882bb43e6e658aaee4cb6865802fa2",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 82,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "unknown",
|
||||
"language_confidence": 0.0,
|
||||
"original": "import logging\n\n\ndef configure(level, handlers, fmt):\n \"\"\"Configure logging for the whole app.\n\n Sets up the root logger with the given level and handlers.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n for h in handlers:\n logger.addHandler(h)\n logger.info('configured')\n return logger\n",
|
||||
"original_tokens": 82,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:12:54.677594+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "full",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n",
|
||||
"input_sha256": "d72fde8ea01cb782c6e3de2e9d8322b40cc5e27593eefbd1049b64ad4467c154",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import logging\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n # [9 lines omitted]\n pass\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n # [1 lines omitted]\n pass",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 148,
|
||||
"compression_ratio": 0.6788990825688074,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n",
|
||||
"original_tokens": 218,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 2,
|
||||
"symbol_scores": {
|
||||
"configure": 0.5,
|
||||
"teardown": 0.5
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.356466+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "This is just a paragraph of plain English prose that contains no recognizable source code constructs at all, so the language detector should classify it as unknown and the compressor should pass it through unchanged without attempting any AST based compression here.\n",
|
||||
"input_sha256": "da21f7240cb20e8b65bee5b06242e82302ace2ad72e430bbcdca59964d6e6274",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "This is just a paragraph of plain English prose that contains no recognizable source code constructs at all, so the language detector should classify it as unknown and the compressor should pass it through unchanged without attempting any AST based compression here.\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 66,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "unknown",
|
||||
"language_confidence": 0.0,
|
||||
"original": "This is just a paragraph of plain English prose that contains no recognizable source code constructs at all, so the language detector should classify it as unknown and the compressor should pass it through unchanged without attempting any AST based compression here.\n",
|
||||
"original_tokens": 66,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.336562+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n",
|
||||
"input_sha256": "e79625ba83a6f22d3bd5d25aa779d38fa07291a4169a5d23fcf2e087d629121c",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import logging\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\"\"\"\n logger = logging.getLogger()\n # [9 lines omitted]\n pass\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n # [1 lines omitted]\n pass",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 96,
|
||||
"compression_ratio": 0.44036697247706424,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import logging\n\n\ndef configure(level, handlers, fmt, propagate, capture_warnings):\n \"\"\"Configure logging for the whole application.\n\n Sets up the root logger with the given level and handler list,\n applies the format string to every handler, and toggles warning\n capture so library warnings are routed through the logger too.\n \"\"\"\n logger = logging.getLogger()\n logger.setLevel(level)\n formatter = logging.Formatter(fmt)\n for handler in handlers:\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = propagate\n logging.captureWarnings(capture_warnings)\n logger.info(\"logging configured at level %s\", level)\n return logger\n\n\ndef teardown(logger):\n for handler in list(logger.handlers):\n handler.flush()\n handler.close()\n logger.removeHandler(handler)\n return logger\n",
|
||||
"original_tokens": 218,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 2,
|
||||
"symbol_scores": {
|
||||
"configure": 0.5,
|
||||
"teardown": 0.5
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.320627+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n",
|
||||
"input_sha256": "e8ce85f0703b083bca8130c3b32c97eeb453dfc6dfcde0988cbaf8d9fc75f354",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import os\nimport sys\nfrom typing import List, Optional\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n # [7 lines omitted]\n pass\n def reset(self):\n self.count = 0\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n # [4 lines omitted]\n pass\n\nGLOBAL_CONST = 42\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 145,
|
||||
"compression_ratio": 0.7073170731707317,
|
||||
"language": "python",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import os\nimport sys\nfrom typing import List, Optional\n\nGLOBAL_CONST = 42\n\n@dataclass\nclass Processor:\n \"\"\"Process a stream of items efficiently.\"\"\"\n name: str\n count: int = 0\n\n def process(self, items: List[str]) -> List[str]:\n \"\"\"Process a list of items and return cleaned results.\"\"\"\n results = []\n for item in items:\n if not item:\n continue\n processed = item.strip().lower()\n results.append(processed)\n self.count += 1\n return results\n\n def reset(self):\n self.count = 0\n\n\ndef standalone(x: int, y: int) -> int:\n total = 0\n for i in range(x):\n for j in range(y):\n total += i * j\n return total\n\n\nif __name__ == \"__main__\":\n p = Processor(\"main\")\n print(p.process([\"a\", \"b\"]))\n",
|
||||
"original_tokens": 205,
|
||||
"preserved_imports": 3,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 1.0,
|
||||
"process": 0.5,
|
||||
"reset": 0.0,
|
||||
"standalone": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.318760+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n",
|
||||
"input_sha256": "f8a7f62203b3d8ba9970fff408a9bfd628d0dbf0b67678f89b0ffb60bbc23d5c",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n // [1 lines omitted; calls: Processor, new, process]\n}\n\n#[derive(Debug)]",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 181,
|
||||
"compression_ratio": 1.0168539325842696,
|
||||
"language": "rust",
|
||||
"language_confidence": 1.0,
|
||||
"original": "use std::collections::HashMap;\n\npub struct Processor {\n name: String,\n count: u32,\n}\n\nimpl Processor {\n pub fn new(name: String) -> Self {\n Self { name, count: 0 }\n }\n\n pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n let mut results = Vec::new();\n for item in items {\n if item.is_empty() {\n continue;\n }\n let clean = item.trim().to_lowercase();\n results.push(clean);\n self.count += 1;\n }\n results\n }\n}\n\n#[derive(Debug)]\nenum Color {\n Red,\n Green,\n Blue,\n}\n\nfn main() {\n let mut p = Processor::new(\"main\".to_string());\n println!(\"{:?}\", p.process(vec![]));\n}\n",
|
||||
"original_tokens": 178,
|
||||
"preserved_imports": 1,
|
||||
"preserved_signatures": 1,
|
||||
"symbol_scores": {
|
||||
"Processor": 1.0,
|
||||
"main": 0.0,
|
||||
"new": 0.333,
|
||||
"process": 0.0
|
||||
},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.330326+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"config": {
|
||||
"ccr_ttl": 300,
|
||||
"compress_comments": true,
|
||||
"docstring_mode": "first_line",
|
||||
"enable_ccr": false,
|
||||
"fallback_to_kompress": false,
|
||||
"language_hint": null,
|
||||
"max_body_lines": 5,
|
||||
"min_tokens_for_compression": 100,
|
||||
"preserve_decorators": true,
|
||||
"preserve_imports": true,
|
||||
"preserve_signatures": true,
|
||||
"preserve_type_annotations": true,
|
||||
"semantic_analysis": true,
|
||||
"target_compression_rate": 0.2
|
||||
},
|
||||
"input": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe<T> = T | null;\n\nexport function lookup(users: User[], id: number): Maybe<User> {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository<T> {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n",
|
||||
"input_sha256": "f9a7c264592ff8c4f726600ef7a89beb9ef1a625cad83a6d1700f09466778889",
|
||||
"output": {
|
||||
"cache_key": null,
|
||||
"compressed": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe<T> = T | null;\n\nexport function lookup(users: User[], id: number): Maybe<User> {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository<T> {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n",
|
||||
"compressed_bodies": 0,
|
||||
"compressed_tokens": 129,
|
||||
"compression_ratio": 1.0,
|
||||
"language": "typescript",
|
||||
"language_confidence": 1.0,
|
||||
"original": "import { Foo } from './foo';\n\ninterface User {\n id: number;\n name: string;\n email?: string;\n}\n\ntype Maybe<T> = T | null;\n\nexport function lookup(users: User[], id: number): Maybe<User> {\n for (const user of users) {\n if (user.id === id) {\n return user;\n }\n }\n return null;\n}\n\nclass Repository<T> {\n private items: T[] = [];\n\n add(item: T): void {\n this.items.push(item);\n if (this.items.length > 1000) {\n this.items.shift();\n }\n }\n}\n",
|
||||
"original_tokens": 129,
|
||||
"preserved_imports": 0,
|
||||
"preserved_signatures": 0,
|
||||
"symbol_scores": {},
|
||||
"syntax_valid": true
|
||||
},
|
||||
"recorded_at": "2026-06-19T00:15:16.325460+00:00",
|
||||
"transform": "code_aware_compressor"
|
||||
}
|
||||
|
|
@ -294,6 +294,20 @@ def record_all(root: Path | None = None) -> dict[str, str]:
|
|||
except Exception as e:
|
||||
statuses["kompress"] = f"blocked:{e.__class__.__name__}:{e}"
|
||||
|
||||
# --- code_aware_compressor ---------------------------------------------
|
||||
# AST code compressor. The workload driver installs an individual-grammar
|
||||
# parser patch (the installed tree-sitter-language-pack is an
|
||||
# API-incompatible native binding) and constructs it with enable_ccr=False
|
||||
# + fallback_to_kompress=False so output is deterministic and
|
||||
# store/model-independent.
|
||||
try:
|
||||
from headroom.transforms.code_compressor import CodeAwareCompressor
|
||||
|
||||
_wrap_method(CodeAwareCompressor, "compress", "code_aware_compressor", root=root)
|
||||
statuses["code_aware_compressor"] = "patched"
|
||||
except Exception as e:
|
||||
statuses["code_aware_compressor"] = f"blocked:{e.__class__.__name__}:{e}"
|
||||
|
||||
return statuses
|
||||
|
||||
|
||||
|
|
@ -765,6 +779,7 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]:
|
|||
"ccr": 0,
|
||||
"content_detector": 0,
|
||||
"kompress": 0,
|
||||
"code_aware_compressor": 0,
|
||||
}
|
||||
|
||||
# log_compressor
|
||||
|
|
@ -871,9 +886,439 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]:
|
|||
except Exception as e:
|
||||
LOG.warning("kompress workload failed: %s", e)
|
||||
|
||||
# code_aware_compressor — AST code compression over all 8 languages.
|
||||
# Installs the individual-grammar parser patch first (see
|
||||
# install_individual_grammar_parsers); enable_ccr=False +
|
||||
# fallback_to_kompress=False keep output deterministic. Soft-fails when
|
||||
# the per-language grammar wheels aren't installed.
|
||||
try:
|
||||
from headroom.transforms.code_compressor import (
|
||||
CodeAwareCompressor,
|
||||
CodeCompressorConfig,
|
||||
)
|
||||
|
||||
install_individual_grammar_parsers()
|
||||
cac = CodeAwareCompressor(
|
||||
CodeCompressorConfig(enable_ccr=False, fallback_to_kompress=False)
|
||||
)
|
||||
for s in _varied_code_inputs():
|
||||
cac.compress(s)
|
||||
counts["code_aware_compressor"] += 1
|
||||
except Exception as e:
|
||||
LOG.warning("code_aware_compressor workload failed: %s", e)
|
||||
|
||||
return counts
|
||||
|
||||
|
||||
def install_individual_grammar_parsers() -> None:
|
||||
"""Repoint `code_compressor._get_parser` at the individual
|
||||
`tree-sitter-<lang>` grammar wheels (pinned to match the Rust crates).
|
||||
|
||||
The installed `tree-sitter-language-pack` is an alef-generated native
|
||||
binding whose `get_language()` returns a non-`tree_sitter.Language` and
|
||||
whose `parse()` wants `str` — API-incompatible with `code_compressor.py`
|
||||
(which builds a stock `tree_sitter.Parser` from a `tree_sitter.Language`).
|
||||
To record fixtures against grammars the Rust port can match byte-for-byte,
|
||||
we swap in stock `tree_sitter.Parser`s bound to the per-language grammars.
|
||||
Raises ImportError when the grammar wheels / core binding aren't present
|
||||
(the workload driver soft-fails, like kompress without its model).
|
||||
"""
|
||||
import tree_sitter_c
|
||||
import tree_sitter_cpp
|
||||
import tree_sitter_go
|
||||
import tree_sitter_java
|
||||
import tree_sitter_javascript
|
||||
import tree_sitter_python
|
||||
import tree_sitter_rust
|
||||
import tree_sitter_typescript
|
||||
from tree_sitter import Language, Parser
|
||||
|
||||
from headroom.transforms import code_compressor as cc
|
||||
|
||||
langs = {
|
||||
"python": Language(tree_sitter_python.language()),
|
||||
"javascript": Language(tree_sitter_javascript.language()),
|
||||
"typescript": Language(tree_sitter_typescript.language_typescript()),
|
||||
"go": Language(tree_sitter_go.language()),
|
||||
"rust": Language(tree_sitter_rust.language()),
|
||||
"java": Language(tree_sitter_java.language()),
|
||||
"c": Language(tree_sitter_c.language()),
|
||||
"cpp": Language(tree_sitter_cpp.language()),
|
||||
}
|
||||
cache: dict[str, Any] = {}
|
||||
|
||||
def _get_parser(language: str) -> Any:
|
||||
if language not in cache:
|
||||
try:
|
||||
cache[language] = Parser(langs[language])
|
||||
except TypeError: # older binding: assign .language
|
||||
p = Parser()
|
||||
p.language = langs[language]
|
||||
cache[language] = p
|
||||
return cache[language]
|
||||
|
||||
cc._get_parser = _get_parser # type: ignore[assignment] # noqa: SLF001
|
||||
cc._check_tree_sitter_available = lambda: True # type: ignore[assignment] # noqa: SLF001
|
||||
|
||||
|
||||
def _varied_code_inputs() -> list[str]:
|
||||
"""≥20 varied source-code inputs spanning all 8 supported languages.
|
||||
|
||||
Exercises: imports, long-bodied functions (body truncation), classes with
|
||||
multiple methods, decorators, type definitions, top-level code, Python
|
||||
docstring first-line reconstruction, short passthrough (<100 tokens), and
|
||||
an UNKNOWN input (plain prose → passthrough with fallback disabled). All
|
||||
inputs are ASCII (the non-ASCII byte/char slice ambiguity is out of parity
|
||||
scope; see code_compressor.rs module docs)."""
|
||||
python_basic = (
|
||||
"import os\n"
|
||||
"import sys\n"
|
||||
"from typing import List, Optional\n\n"
|
||||
"GLOBAL_CONST = 42\n\n"
|
||||
"@dataclass\n"
|
||||
"class Processor:\n"
|
||||
' """Process a stream of items efficiently."""\n'
|
||||
" name: str\n"
|
||||
" count: int = 0\n\n"
|
||||
" def process(self, items: List[str]) -> List[str]:\n"
|
||||
' """Process a list of items and return cleaned results."""\n'
|
||||
" results = []\n"
|
||||
" for item in items:\n"
|
||||
" if not item:\n"
|
||||
" continue\n"
|
||||
" processed = item.strip().lower()\n"
|
||||
" results.append(processed)\n"
|
||||
" self.count += 1\n"
|
||||
" return results\n\n"
|
||||
" def reset(self):\n"
|
||||
" self.count = 0\n\n\n"
|
||||
"def standalone(x: int, y: int) -> int:\n"
|
||||
" total = 0\n"
|
||||
" for i in range(x):\n"
|
||||
" for j in range(y):\n"
|
||||
" total += i * j\n"
|
||||
" return total\n\n\n"
|
||||
'if __name__ == "__main__":\n'
|
||||
' p = Processor("main")\n'
|
||||
' print(p.process(["a", "b"]))\n'
|
||||
)
|
||||
python_nodoc = (
|
||||
"from collections import defaultdict\n\n\n"
|
||||
"def build_index(records):\n"
|
||||
" index = defaultdict(list)\n"
|
||||
" for rec in records:\n"
|
||||
' key = rec.get("id")\n'
|
||||
" if key is None:\n"
|
||||
" continue\n"
|
||||
" index[key].append(rec)\n"
|
||||
" if len(index[key]) > 100:\n"
|
||||
" index[key] = index[key][:100]\n"
|
||||
" return index\n\n\n"
|
||||
"def merge(a, b):\n"
|
||||
" out = dict(a)\n"
|
||||
" for k, v in b.items():\n"
|
||||
" out[k] = v\n"
|
||||
" return out\n"
|
||||
)
|
||||
python_multiline_ds = (
|
||||
"import logging\n\n\n"
|
||||
"def configure(level, handlers, fmt, propagate, capture_warnings):\n"
|
||||
' """Configure logging for the whole application.\n\n'
|
||||
" Sets up the root logger with the given level and handler list,\n"
|
||||
" applies the format string to every handler, and toggles warning\n"
|
||||
" capture so library warnings are routed through the logger too.\n"
|
||||
' """\n'
|
||||
" logger = logging.getLogger()\n"
|
||||
" logger.setLevel(level)\n"
|
||||
" formatter = logging.Formatter(fmt)\n"
|
||||
" for handler in handlers:\n"
|
||||
" handler.setFormatter(formatter)\n"
|
||||
" logger.addHandler(handler)\n"
|
||||
" logger.propagate = propagate\n"
|
||||
" logging.captureWarnings(capture_warnings)\n"
|
||||
' logger.info("logging configured at level %s", level)\n'
|
||||
" return logger\n\n\n"
|
||||
"def teardown(logger):\n"
|
||||
" for handler in list(logger.handlers):\n"
|
||||
" handler.flush()\n"
|
||||
" handler.close()\n"
|
||||
" logger.removeHandler(handler)\n"
|
||||
" return logger\n"
|
||||
)
|
||||
javascript_basic = (
|
||||
"import { foo } from './foo';\n"
|
||||
"const bar = require('bar');\n\n"
|
||||
"const CONST = 99;\n\n"
|
||||
"export function processData(items) {\n"
|
||||
" const results = [];\n"
|
||||
" for (const item of items) {\n"
|
||||
" if (!item) {\n"
|
||||
" continue;\n"
|
||||
" }\n"
|
||||
" const clean = item.trim().toLowerCase();\n"
|
||||
" results.push(clean);\n"
|
||||
" }\n"
|
||||
" return results;\n"
|
||||
"}\n\n"
|
||||
"class Widget {\n"
|
||||
" constructor(name) {\n"
|
||||
" this.name = name;\n"
|
||||
" this.count = 0;\n"
|
||||
" }\n\n"
|
||||
" render(ctx) {\n"
|
||||
" ctx.clear();\n"
|
||||
" ctx.draw(this.name);\n"
|
||||
" this.count += 1;\n"
|
||||
" return ctx;\n"
|
||||
" }\n"
|
||||
"}\n\n"
|
||||
"module.exports = { processData, Widget };\n"
|
||||
)
|
||||
typescript_basic = (
|
||||
"import { Foo } from './foo';\n\n"
|
||||
"interface User {\n"
|
||||
" id: number;\n"
|
||||
" name: string;\n"
|
||||
" email?: string;\n"
|
||||
"}\n\n"
|
||||
"type Maybe<T> = T | null;\n\n"
|
||||
"export function lookup(users: User[], id: number): Maybe<User> {\n"
|
||||
" for (const user of users) {\n"
|
||||
" if (user.id === id) {\n"
|
||||
" return user;\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
" return null;\n"
|
||||
"}\n\n"
|
||||
"class Repository<T> {\n"
|
||||
" private items: T[] = [];\n\n"
|
||||
" add(item: T): void {\n"
|
||||
" this.items.push(item);\n"
|
||||
" if (this.items.length > 1000) {\n"
|
||||
" this.items.shift();\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
)
|
||||
go_basic = (
|
||||
"package main\n\n"
|
||||
"import (\n"
|
||||
'\t"fmt"\n'
|
||||
'\t"strings"\n'
|
||||
")\n\n"
|
||||
"type Processor struct {\n"
|
||||
"\tName string\n"
|
||||
"\tCount int\n"
|
||||
"}\n\n"
|
||||
"func (p *Processor) Process(items []string) []string {\n"
|
||||
"\tresults := make([]string, 0, len(items))\n"
|
||||
"\tfor _, item := range items {\n"
|
||||
'\t\tif item == "" {\n'
|
||||
"\t\t\tcontinue\n"
|
||||
"\t\t}\n"
|
||||
"\t\tclean := strings.ToLower(strings.TrimSpace(item))\n"
|
||||
"\t\tresults = append(results, clean)\n"
|
||||
"\t\tp.Count++\n"
|
||||
"\t}\n"
|
||||
"\treturn results\n"
|
||||
"}\n\n"
|
||||
"func main() {\n"
|
||||
'\tp := &Processor{Name: "main"}\n'
|
||||
'\tfmt.Println(p.Process([]string{"a", "b"}))\n'
|
||||
"}\n"
|
||||
)
|
||||
rust_basic = (
|
||||
"use std::collections::HashMap;\n\n"
|
||||
"pub struct Processor {\n"
|
||||
" name: String,\n"
|
||||
" count: u32,\n"
|
||||
"}\n\n"
|
||||
"impl Processor {\n"
|
||||
" pub fn new(name: String) -> Self {\n"
|
||||
" Self { name, count: 0 }\n"
|
||||
" }\n\n"
|
||||
" pub fn process(&mut self, items: Vec<String>) -> Vec<String> {\n"
|
||||
" let mut results = Vec::new();\n"
|
||||
" for item in items {\n"
|
||||
" if item.is_empty() {\n"
|
||||
" continue;\n"
|
||||
" }\n"
|
||||
" let clean = item.trim().to_lowercase();\n"
|
||||
" results.push(clean);\n"
|
||||
" self.count += 1;\n"
|
||||
" }\n"
|
||||
" results\n"
|
||||
" }\n"
|
||||
"}\n\n"
|
||||
"#[derive(Debug)]\n"
|
||||
"enum Color {\n"
|
||||
" Red,\n"
|
||||
" Green,\n"
|
||||
" Blue,\n"
|
||||
"}\n\n"
|
||||
"fn main() {\n"
|
||||
' let mut p = Processor::new("main".to_string());\n'
|
||||
' println!("{:?}", p.process(vec![]));\n'
|
||||
"}\n"
|
||||
)
|
||||
java_basic = (
|
||||
"package com.example;\n\n"
|
||||
"import java.util.List;\n"
|
||||
"import java.util.ArrayList;\n\n"
|
||||
"public class Processor {\n"
|
||||
" private String name;\n"
|
||||
" private int count;\n\n"
|
||||
" public Processor(String name) {\n"
|
||||
" this.name = name;\n"
|
||||
" this.count = 0;\n"
|
||||
" }\n\n"
|
||||
" @Override\n"
|
||||
" public String toString() {\n"
|
||||
' return "Processor(" + name + ")";\n'
|
||||
" }\n\n"
|
||||
" public List<String> process(List<String> items) {\n"
|
||||
" List<String> results = new ArrayList<>();\n"
|
||||
" for (String item : items) {\n"
|
||||
" if (item == null || item.isEmpty()) {\n"
|
||||
" continue;\n"
|
||||
" }\n"
|
||||
" String clean = item.trim().toLowerCase();\n"
|
||||
" results.add(clean);\n"
|
||||
" this.count += 1;\n"
|
||||
" }\n"
|
||||
" return results;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
)
|
||||
c_basic = (
|
||||
"#include <stdio.h>\n"
|
||||
"#include <stdlib.h>\n"
|
||||
"#include <string.h>\n\n"
|
||||
"typedef struct {\n"
|
||||
" char name[64];\n"
|
||||
" int count;\n"
|
||||
"} Processor;\n\n"
|
||||
"int process(Processor *p, const char **items, int n) {\n"
|
||||
" int kept = 0;\n"
|
||||
" for (int i = 0; i < n; i++) {\n"
|
||||
" if (items[i] == NULL || strlen(items[i]) == 0) {\n"
|
||||
" continue;\n"
|
||||
" }\n"
|
||||
" kept++;\n"
|
||||
" p->count++;\n"
|
||||
" }\n"
|
||||
" return kept;\n"
|
||||
"}\n\n"
|
||||
"int main(void) {\n"
|
||||
' Processor p = {"main", 0};\n'
|
||||
' const char *items[] = {"a", "b"};\n'
|
||||
' printf("%d\\n", process(&p, items, 2));\n'
|
||||
" return 0;\n"
|
||||
"}\n"
|
||||
)
|
||||
cpp_basic = (
|
||||
"#include <iostream>\n"
|
||||
"#include <vector>\n"
|
||||
"#include <string>\n\n"
|
||||
"namespace app {\n\n"
|
||||
"class Processor {\n"
|
||||
"public:\n"
|
||||
" Processor(const std::string &name) : name_(name), count_(0) {}\n\n"
|
||||
" std::vector<std::string> process(const std::vector<std::string> &items) {\n"
|
||||
" std::vector<std::string> results;\n"
|
||||
" for (const auto &item : items) {\n"
|
||||
" if (item.empty()) {\n"
|
||||
" continue;\n"
|
||||
" }\n"
|
||||
" results.push_back(item);\n"
|
||||
" count_++;\n"
|
||||
" }\n"
|
||||
" return results;\n"
|
||||
" }\n\n"
|
||||
"private:\n"
|
||||
" std::string name_;\n"
|
||||
" int count_;\n"
|
||||
"};\n\n"
|
||||
"} // namespace app\n\n"
|
||||
"int main() {\n"
|
||||
' app::Processor p("main");\n'
|
||||
" return 0;\n"
|
||||
"}\n"
|
||||
)
|
||||
# Short (<100 char/4 tokens) → passthrough, per a couple languages.
|
||||
short_py = "def f(x):\n return x + 1\n"
|
||||
short_js = "const a = 1;\nfunction g() { return a; }\n"
|
||||
# UNKNOWN → passthrough (fallback_to_kompress=False in the recorder).
|
||||
unknown_prose = (
|
||||
"This is just a paragraph of plain English prose that contains no "
|
||||
"recognizable source code constructs at all, so the language detector "
|
||||
"should classify it as unknown and the compressor should pass it "
|
||||
"through unchanged without attempting any AST based compression here.\n"
|
||||
)
|
||||
# A longer multi-function Python file to exercise budget allocation.
|
||||
python_orchestrator = (
|
||||
"import json\n\n\n"
|
||||
"def load(path):\n"
|
||||
" with open(path) as fh:\n"
|
||||
" data = json.load(fh)\n"
|
||||
" cleaned = {}\n"
|
||||
" for key, value in data.items():\n"
|
||||
" if value is None:\n"
|
||||
" continue\n"
|
||||
" cleaned[key] = value\n"
|
||||
" return cleaned\n\n\n"
|
||||
"def transform(records, factor):\n"
|
||||
" out = []\n"
|
||||
" for r in records:\n"
|
||||
" scaled = r * factor\n"
|
||||
" if scaled > 1000:\n"
|
||||
" scaled = 1000\n"
|
||||
" out.append(scaled)\n"
|
||||
" return out\n\n\n"
|
||||
"def run(path, factor):\n"
|
||||
" data = load(path)\n"
|
||||
" values = list(data.values())\n"
|
||||
" result = transform(values, factor)\n"
|
||||
" return sum(result)\n"
|
||||
)
|
||||
|
||||
return [
|
||||
python_basic,
|
||||
python_nodoc,
|
||||
python_multiline_ds,
|
||||
python_orchestrator,
|
||||
javascript_basic,
|
||||
typescript_basic,
|
||||
go_basic,
|
||||
rust_basic,
|
||||
java_basic,
|
||||
c_basic,
|
||||
cpp_basic,
|
||||
short_py,
|
||||
short_js,
|
||||
unknown_prose,
|
||||
# Padded variants for >= 20 unique inputs. A trailing comment keeps
|
||||
# each input unique (distinct fixture hash) while preserving the
|
||||
# structural paths exercised above.
|
||||
f"{python_basic}\n# variant 1",
|
||||
f"{javascript_basic}\n// variant 1",
|
||||
f"{go_basic}\n// variant 1",
|
||||
f"{rust_basic}\n// variant 1",
|
||||
f"{java_basic}\n// variant 1",
|
||||
f"{c_basic}\n// variant 1",
|
||||
f"{cpp_basic}\n// variant 1",
|
||||
f"{typescript_basic}\n// variant 1",
|
||||
f"{python_orchestrator}\n# variant 1",
|
||||
]
|
||||
|
||||
|
||||
def _docstring_mode_inputs() -> list[str]:
|
||||
"""Docstring-bearing Python samples, for exercising the non-default
|
||||
`DocstringMode` branches (FULL keeps the whole docstring, REMOVE drops it).
|
||||
The default `_varied_code_inputs` only exercises FIRST_LINE."""
|
||||
return [s for s in _varied_code_inputs() if '"""' in s][:2]
|
||||
|
||||
|
||||
def _varied_kompress_inputs() -> list[str]:
|
||||
"""≥20 varied prose/log/mixed inputs for the Kompress ML compressor.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue