diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py index db8d284e8..dff901a43 100644 --- a/headroom/transforms/smart_crusher.py +++ b/headroom/transforms/smart_crusher.py @@ -1,984 +1,55 @@ -"""Smart statistical tool output compression for Headroom SDK. +"""Smart JSON array crusher — Rust-backed via PyO3. -This module provides intelligent JSON compression based on statistical analysis -rather than fixed rules. It analyzes data patterns and applies optimal compression -strategies to maximize token reduction while preserving important information. +The Python implementation has been retired (Stage 3c.1b, 2026-04-27). +All array compression now goes through `headroom._core.SmartCrusher` +(built from `crates/headroom-py`). Byte-equality of the two +implementations was verified against 17 recorded fixtures +(`tests/parity/fixtures/smart_crusher/`) before the Python source was +removed; the Rust crate has its own coverage in `crates/headroom-core/` +(388 unit tests + property tests). -SCOPE: SmartCrusher handles JSON arrays of ANY type — dicts, strings, numbers, -mixed types, and nested arrays. Non-JSON content (plain text, search results, -logs, code, diffs) passes through UNCHANGED. +This module retains the public surface — `SmartCrusherConfig`, +`CrushResult`, `SmartCrusher`, `smart_crush_tool_output` — so existing +call sites keep working unchanged. The dataclasses are still pure +Python because callers use `asdict()`, `__dict__`, and dataclass +matching on them. Only the `SmartCrusher` class delegates to Rust. -TEXT COMPRESSION IS OPT-IN: For text-based content, Headroom provides standalone -utilities that applications can use explicitly: -- SearchCompressor: For grep/ripgrep output (file:line:content format) -- LogCompressor: For build/test logs (pytest, npm, cargo output) -- Kompress: For generic plain text (ML-based, requires [ml] extra) +The `headroom._core` extension is a hard import: there is no Python +fallback. Build it locally with `scripts/build_rust_extension.sh` +(wraps `maturin develop`) or install a prebuilt wheel. -Applications should decide when and how to use text compression based on their -specific needs. This design prevents lossy text compression from being applied -automatically, which could lose important context in coding tasks. - -SCHEMA-PRESERVING: Output contains only items from the original array. -No wrappers, no generated text, no metadata keys. This ensures downstream -tools and parsers work unchanged. - -Supported JSON types: -- Arrays of dicts: Full statistical analysis with adaptive K (Kneedle algorithm) -- Arrays of strings: Dedup + adaptive sampling + error preservation -- Arrays of numbers: Statistical summary + outlier/change-point preservation -- Mixed-type arrays: Grouped by type, each group compressed independently -- Flat objects (many keys): Key-level adaptive sampling -- Nested objects: Recursive compression of inner arrays/objects - -Safety guarantees (consistent across ALL types): -- First K, last K items always kept (K is adaptive, not hardcoded) -- Error items (containing 'error', 'exception', 'failed', 'critical') never dropped -- Anomalous numeric items (> 2 std from mean) always kept -- Items around detected change points preserved -- Items with high relevance score to user query (via RelevanceScorer) - -Key Features: -- RelevanceScorer: ML-powered or BM25-based relevance matching (replaces regex) -- Variance-based change point detection (preserve anomalies) -- Error item detection (never lose error messages) -- Pattern detection (time series, logs, search results) -- Strategy selection based on data characteristics +Stage 3c.1 deliberately keeps the optional subsystems (TOIN, +feedback, CCR marker injection, telemetry) disabled in the Rust port. +The shim accepts `relevance_config`, `scorer`, and `ccr_config` +constructor args for source compatibility but does not wire them +through — they re-attach in Stage 3c.2 when those subsystems land in +Rust. CCR marker injection in `_smart_crush_content` is a Stage 3c.2 +follow-up; today the Rust port never emits CCR markers, so the +disabled-path behavior is byte-equal. """ from __future__ import annotations -import hashlib -import json import logging -import math -import re -import statistics -import threading -from collections import Counter -from dataclasses import dataclass, field -from enum import Enum +from dataclasses import dataclass from typing import Any -from ..cache.compression_feedback import CompressionFeedback, get_compression_feedback -from ..cache.compression_store import CompressionStore, get_compression_store -from ..config import AnchorConfig, CCRConfig, RelevanceScorerConfig, TransformResult -from ..relevance import RelevanceScorer, create_scorer -from ..telemetry import TelemetryCollector, ToolSignature, get_telemetry_collector -from ..telemetry.models import FieldSemantics -from ..telemetry.toin import ToolIntelligenceNetwork, get_toin +from ..config import CCRConfig, TransformResult from ..tokenizer import Tokenizer -from ..utils import ( - compute_short_hash, - create_tool_digest_marker, - deep_copy_messages, - safe_json_dumps, - safe_json_loads, -) -from .anchor_selector import AnchorSelector -from .anchor_selector import DataPattern as AnchorDataPattern +from ..utils import compute_short_hash, create_tool_digest_marker, deep_copy_messages from .base import Transform -from .error_detection import ERROR_KEYWORDS logger = logging.getLogger(__name__) -# Legacy patterns for backwards compatibility (extract_query_anchors) -_UUID_PATTERN = re.compile( - r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" -) -_NUMERIC_ID_PATTERN = re.compile(r"\b\d{4,}\b") # 4+ digit numbers (likely IDs) -_HOSTNAME_PATTERN = re.compile( - r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b" -) -_QUOTED_STRING_PATTERN = re.compile(r"['\"]([^'\"]{1,50})['\"]") # Short quoted strings -_EMAIL_PATTERN = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b") -# Temporal detection patterns (compiled once, used in SmartAnalyzer._detect_temporal_field) -_ISO_DATETIME_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}") -_ISO_DATE_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}$") - - -def extract_query_anchors(text: str) -> set[str]: - """Extract query anchors from user text (legacy regex-based method). - - DEPRECATED: Use RelevanceScorer.score_batch() for better semantic matching. - - Query anchors are identifiers or values that the user is likely searching for. - When crushing tool outputs, items matching these anchors should be preserved. - - Extracts: - - UUIDs (e.g., "550e8400-e29b-41d4-a716-446655440000") - - Numeric IDs (4+ digits, e.g., "12345", "1001234") - - Hostnames (e.g., "api.example.com", "server-01.prod") - - Quoted strings (e.g., 'Alice', "error_code") - - Email addresses (e.g., "user@example.com") - - Args: - text: User message text to extract anchors from. - - Returns: - Set of anchor strings (lowercased for case-insensitive matching). - """ - anchors: set[str] = set() - - if not text: - return anchors - - # UUIDs - for match in _UUID_PATTERN.findall(text): - anchors.add(match.lower()) - - # Numeric IDs - for match in _NUMERIC_ID_PATTERN.findall(text): - anchors.add(match) - - # Hostnames - for match in _HOSTNAME_PATTERN.findall(text): - # Filter out common false positives - if match.lower() not in ("e.g", "i.e", "etc."): - anchors.add(match.lower()) - - # Quoted strings - for match in _QUOTED_STRING_PATTERN.findall(text): - if len(match.strip()) >= 2: # Skip very short matches - anchors.add(match.lower()) - - # Email addresses - for match in _EMAIL_PATTERN.findall(text): - anchors.add(match.lower()) - - return anchors - - -def item_matches_anchors(item: dict, anchors: set[str]) -> bool: - """Check if an item matches any query anchors (legacy method). - - DEPRECATED: Use RelevanceScorer for better matching. - - Args: - item: Dictionary item from tool output. - anchors: Set of anchor strings to match. - - Returns: - True if any anchor is found in the item's string representation. - """ - if not anchors: - return False - - item_str = str(item).lower() - return any(anchor in item_str for anchor in anchors) - - -def _hash_field_name(field_name: str) -> str: - """Hash a field name to match TOIN's anonymized preserve_fields. - - TOIN stores field names as SHA256[:8] hashes for privacy. - This function produces the same hash format. - """ - return hashlib.sha256(field_name.encode()).hexdigest()[:8] - - -# Minimum chars for a text field to be worth compressing within an item -_MIN_FIELD_CHARS_FOR_WITHIN = 200 - -# Lazy-loaded compressor for within-item text compression (thread-safe) -_within_compressor: Any = None -_within_compressor_checked = False -_within_compressor_lock = threading.Lock() - - -def _get_within_compressor() -> Any: - """Get a text compressor for within-item field compression. - - Returns Kompress if available (requires [ml] extra), else None. - Thread-safe via double-checked locking. - """ - global _within_compressor, _within_compressor_checked - if not _within_compressor_checked: - with _within_compressor_lock: - if not _within_compressor_checked: - try: - from .kompress_compressor import KompressCompressor, is_kompress_available - - if is_kompress_available(): - _within_compressor = KompressCompressor() - logger.debug("Within-item compression: using Kompress") - except ImportError: - pass - _within_compressor_checked = True - return _within_compressor - - -def _compress_text_within_items(items: list[dict], context: str = "") -> list[dict]: - """Compress long text fields WITHIN each item, keeping all items. - - Used when diversity is high (all items are unique) — instead of dropping - items, compress the verbose text inside each one. Falls back to the - original list unchanged if no compressor is available or no field is - long enough to benefit. - - Args: - items: JSON-parsed list of dicts. - context: User query context for relevance-aware compression. - - Returns: - Compressed items (new list) or the *same* ``items`` object if - nothing was compressed (caller checks identity). - """ - compressor = _get_within_compressor() - if compressor is None: - return items # No ML compressor available — pass through - - any_compressed = False - result: list[dict] = [] - - for item in items: - new_item = dict(item) # Shallow copy - item_changed = False - - for key, value in item.items(): - if not isinstance(value, str) or len(value) < _MIN_FIELD_CHARS_FOR_WITHIN: - continue - - try: - compressed = compressor.compress(value, context=context) - if compressed.compressed and len(compressed.compressed) < len(value) * 0.9: - new_item[key] = compressed.compressed - item_changed = True - except Exception: - pass # Compression failed for this field — keep original - - result.append(new_item if item_changed else item) - if item_changed: - any_compressed = True - - return result if any_compressed else items - - -def _get_preserve_field_values( - item: dict, - preserve_field_hashes: list[str], -) -> list[tuple[str, Any]]: - """Get values from item fields that match TOIN's preserve_field hashes. - - TOIN stores preserve_fields as hashed field names (SHA256[:8]). - This function iterates over item fields, hashes each, and returns - matching field names and values. - - Args: - item: Dictionary item from tool output. - preserve_field_hashes: List of SHA256[:8] hashed field names from TOIN. - - Returns: - List of (field_name, value) tuples for fields that match. - """ - if not preserve_field_hashes or not item: - return [] - - # Convert preserve_fields to set for O(1) lookup - hash_set = set(preserve_field_hashes) - - matches = [] - for field_name, value in item.items(): - field_hash = _hash_field_name(field_name) - if field_hash in hash_set: - matches.append((field_name, value)) - - return matches - - -def _percentile_linear(sorted_values: list[float], q: float) -> float: - """Linear-interpolation percentile, matching numpy's "linear" method. - - BUG #1 FIX (replaces integer-division indexing in _crush_number_array - that was off by one for `len < 8`). Computes: - index = q * (n - 1) - if index is integer: return sorted_values[index] - else: linear interpolate between floor and ceil - - Args: - sorted_values: Pre-sorted ascending list of finite numbers. - q: Quantile in [0, 1] (e.g. 0.25 for p25, 0.75 for p75). - - Returns: - The interpolated quantile value. Empty input returns 0.0. - """ - n = len(sorted_values) - if n == 0: - return 0.0 - if n == 1: - return float(sorted_values[0]) - pos = q * (n - 1) - lo = int(pos) - hi = lo + 1 if lo + 1 < n else lo - frac = pos - lo - return sorted_values[lo] * (1 - frac) + sorted_values[hi] * frac - - -def _item_has_preserve_field_match( - item: dict, - preserve_field_hashes: list[str], - query_context: str, -) -> bool: - """Check if item has a preserve_field value that matches query context. - - Args: - item: Dictionary item from tool output. - preserve_field_hashes: List of SHA256[:8] hashed field names from TOIN. - query_context: User's query to match against field values. - - Returns: - True if any preserve_field value matches the query context. - """ - if not query_context: - return False - - query_lower = query_context.lower() - - for _field_name, value in _get_preserve_field_values(item, preserve_field_hashes): - if value is not None: - value_str = str(value).lower() - if value_str in query_lower or query_lower in value_str: - return True - - return False - - -class CompressionStrategy(Enum): - """Compression strategies based on data patterns.""" - - NONE = "none" # No compression needed - SKIP = "skip" # Explicitly skip - not safe to crush - TIME_SERIES = "time_series" # Keep change points, summarize stable - CLUSTER_SAMPLE = "cluster" # Dedupe similar items - TOP_N = "top_n" # Keep highest scored items - SMART_SAMPLE = "smart_sample" # Statistical sampling with constants - - -class ArrayType(Enum): - """JSON array element type classification.""" - - DICT_ARRAY = "dict_array" # [{...}, {...}, ...] - STRING_ARRAY = "string_array" # ["a", "b", "c", ...] - NUMBER_ARRAY = "number_array" # [1, 2.5, 3, ...] - BOOL_ARRAY = "bool_array" # [true, false, ...] - NESTED_ARRAY = "nested_array" # [[...], [...], ...] - MIXED_ARRAY = "mixed_array" # [{"a":1}, "str", 42, ...] - EMPTY = "empty" - - -def _classify_array(items: list) -> ArrayType: - """Classify a JSON array by its element types. - - Uses set-of-types check on ALL elements (not sampling) to guarantee - correct classification. Fast because type() is O(1). - """ - if not items: - return ArrayType.EMPTY - # Note: bool is a subclass of int in Python, so check bool first - types = set() - has_bool = False - for item in items: - if isinstance(item, bool): - has_bool = True - types.add(type(item)) - if has_bool and types <= {bool, int}: - # All bools (Python's True/False are int subclass) - if all(isinstance(i, bool) for i in items): - return ArrayType.BOOL_ARRAY - if types == {dict}: - return ArrayType.DICT_ARRAY - if types == {str}: - return ArrayType.STRING_ARRAY - if types <= {int, float} and not has_bool: - return ArrayType.NUMBER_ARRAY - if types == {list}: - return ArrayType.NESTED_ARRAY - return ArrayType.MIXED_ARRAY - - -# ===================================================================== -# STATISTICAL FIELD DETECTION (replaces hardcoded string patterns) -# ===================================================================== -# Instead of matching field names like "id", "score", "error", we use -# statistical and structural properties of the data to detect field types. - - -def _is_uuid_format(value: str) -> bool: - """Check if a string looks like a UUID (structural pattern).""" - if not isinstance(value, str) or len(value) != 36: - return False - # UUID format: 8-4-4-4-12 hex chars - parts = value.split("-") - if len(parts) != 5: - return False - expected_lens = [8, 4, 4, 4, 12] - for part, expected_len in zip(parts, expected_lens): - if len(part) != expected_len: - return False - if not all(c in "0123456789abcdefABCDEF" for c in part): - return False - return True - - -def _calculate_string_entropy(s: str) -> float: - """Calculate Shannon entropy of a string, normalized to [0, 1]. - - High entropy (>0.7) suggests random/ID-like content. - Low entropy (<0.3) suggests repetitive/predictable content. - """ - if not s or len(s) < 2: - return 0.0 - - # Count character frequencies - freq: dict[str, int] = {} - for c in s: - freq[c] = freq.get(c, 0) + 1 - - # Calculate entropy - import math - - entropy = 0.0 - length = len(s) - for count in freq.values(): - p = count / length - if p > 0: - entropy -= p * math.log2(p) - - # Normalize by max possible entropy for this length - max_entropy = math.log2(min(len(freq), length)) - if max_entropy > 0: - return entropy / max_entropy - return 0.0 - - -def _detect_sequential_pattern(values: list[Any], check_order: bool = True) -> bool: - """Detect if numeric values form a sequential pattern (like IDs: 1,2,3,...). - - Returns True if values appear to be auto-incrementing or sequential. - - Args: - values: List of values to check. - check_order: If True, also check if values are in ascending order in the array. - Score fields are often sorted descending, while IDs are ascending. - """ - if len(values) < 5: - return False - - # BUG #2 FIX: track whether ANY value was already numeric (not - # just a stringified number). If every "number" came from `int(s)` - # of a string value, the field is almost certainly a categorical - # string ID — possibly zero-padded — and should NOT be flagged - # as sequential. Without this guard, `["001", "002", "003"]` - # silently becomes `[1, 2, 3]` and gets misclassified. - nums = [] - had_non_string_numeric = False - for v in values: - if isinstance(v, int | float) and not isinstance(v, bool): - nums.append(v) - had_non_string_numeric = True - elif isinstance(v, str): - try: - nums.append(int(v)) - # Intentionally do NOT set had_non_string_numeric — - # see fix doc above. - except ValueError: - pass - - if len(nums) < 5: - return False - - # BUG #2 FIX: if every parseable value came from a string, the - # field is categorical (zero-padded codes, alphanumeric IDs that - # happen to be int-parseable, etc.). Don't classify as sequential. - if not had_non_string_numeric: - return False - - # Need at least 2 elements for pairwise comparison - if len(nums) < 2: - return False - - # Check if sorted values form a near-sequence - sorted_nums = sorted(nums) - diffs = [sorted_nums[i + 1] - sorted_nums[i] for i in range(len(sorted_nums) - 1)] - - if not diffs: - return False - - # If most differences are 1 (or small constant), it's sequential - avg_diff = sum(diffs) / len(diffs) - if 0.5 <= avg_diff <= 2.0: - # Check consistency - sequential IDs have consistent spacing - consistent_count = sum(1 for d in diffs if 0.5 <= d <= 2.0) - is_sequential = consistent_count / len(diffs) > 0.8 - - # Additional check: IDs are typically in ASCENDING order in the array - # Scores sorted by relevance are typically in DESCENDING order - if check_order and is_sequential: - # Check if original order is ascending (like IDs) - ascending_count = sum(1 for i in range(len(nums) - 1) if nums[i] <= nums[i + 1]) - is_ascending = ascending_count / (len(nums) - 1) > 0.7 - return is_ascending # Only flag as sequential if ascending (ID-like) - - return is_sequential - - return False - - -def _detect_id_field_statistically(stats: FieldStats, values: list[Any]) -> tuple[bool, float]: - """Detect if a field is an ID field using statistical properties. - - Returns (is_id_field, confidence). - - ID fields have: - - Very high uniqueness (>0.95) - - Sequential numeric pattern OR UUID format OR high entropy strings - """ - # Must have high uniqueness - if stats.unique_ratio < 0.9: - return False, 0.0 - - confidence = 0.0 - - # Check for UUID format (structural detection) - if stats.field_type == "string": - sample_values = [v for v in values[:20] if isinstance(v, str)] - uuid_count = sum(1 for v in sample_values if _is_uuid_format(v)) - if sample_values and uuid_count / len(sample_values) > 0.8: - return True, 0.95 - - # Check for high entropy (random string IDs) - if sample_values: - avg_entropy = sum(_calculate_string_entropy(v) for v in sample_values) / len( - sample_values - ) - if avg_entropy > 0.7 and stats.unique_ratio > 0.95: - confidence = 0.8 - return True, confidence - - # Check for sequential numeric pattern - if stats.field_type == "numeric": - if _detect_sequential_pattern(values) and stats.unique_ratio > 0.95: - return True, 0.9 - - # High uniqueness numeric with high range suggests ID - if stats.min_val is not None and stats.max_val is not None: - value_range = stats.max_val - stats.min_val - if value_range > 0 and stats.unique_ratio > 0.95: - return True, 0.85 - - # Very high uniqueness alone is a signal (even without other patterns) - if stats.unique_ratio > 0.98: - return True, 0.7 - - return False, 0.0 - - -def _detect_score_field_statistically(stats: FieldStats, items: list[dict]) -> tuple[bool, float]: - """Detect if a field is a score/ranking field using statistical properties. - - Returns (is_score_field, confidence). - - Score fields have: - - Numeric type - - Bounded range (0-1, 0-10, 0-100, or similar) - - NOT sequential (unlike IDs) - - Often the data appears sorted by this field (descending) - """ - if stats.field_type != "numeric": - return False, 0.0 - - if stats.min_val is None or stats.max_val is None: - return False, 0.0 - - confidence = 0.0 - - # Check for bounded range typical of scores - min_val, max_val = stats.min_val, stats.max_val - - # Common score ranges: [0,1], [0,10], [0,100], [-1,1], [0,5] - is_bounded = False - if 0 <= min_val <= 1 and 0 <= max_val <= 1: # [0,1] range - is_bounded = True - confidence += 0.4 - elif 0 <= min_val <= 10 and 0 <= max_val <= 10: # [0,10] range - is_bounded = True - confidence += 0.3 - elif 0 <= min_val <= 100 and 0 <= max_val <= 100: # [0,100] range - is_bounded = True - confidence += 0.25 - elif -1 <= min_val and max_val <= 1: # [-1,1] range - is_bounded = True - confidence += 0.35 - - if not is_bounded: - return False, 0.0 - - # Should NOT be sequential (IDs are sequential, scores are not) - sample_values = [item.get(stats.name) for item in items[:50] if stats.name in item] - if _detect_sequential_pattern(sample_values): - return False, 0.0 - - # Check if data appears sorted by this field (descending = relevance sorted) - # Filter out NaN/Inf which break comparisons - values_in_order: list[float] = [] - for item in items: - if stats.name in item: - val = item.get(stats.name) - if isinstance(val, int | float) and math.isfinite(val): - values_in_order.append(float(val)) - if len(values_in_order) >= 5: - # Check for descending sort - num_pairs = len(values_in_order) - 1 - descending_count = sum( - 1 for i in range(num_pairs) if values_in_order[i] >= values_in_order[i + 1] - ) - if num_pairs > 0 and descending_count / num_pairs > 0.7: - confidence += 0.3 - - # Score fields often have floating point values - # Filter out NaN/Inf which can't be converted to int - float_count = sum( - 1 for v in values_in_order[:20] if isinstance(v, float) and math.isfinite(v) and v != int(v) - ) - if float_count > len(values_in_order[:20]) * 0.3: - confidence += 0.1 - - return confidence >= 0.4, min(confidence, 0.95) - - -def _detect_structural_outliers(items: list[dict]) -> list[int]: - """Detect items that are structural outliers (error-like items). - - Instead of looking for "error" keywords, we detect: - 1. Items with extra fields that others don't have - 2. Items with rare status/state values - 3. Items with significantly different structure - - Returns indices of outlier items. - """ - if len(items) < 5: - return [] - - outlier_indices: list[int] = [] - - # 1. Detect items with extra fields - # Find the "common" field set (fields present in >80% of items) - field_counts: dict[str, int] = {} - for item in items: - if isinstance(item, dict): - for key in item.keys(): - field_counts[key] = field_counts.get(key, 0) + 1 - - n = len(items) - common_fields = {k for k, v in field_counts.items() if v >= n * 0.8} - rare_fields = {k for k, v in field_counts.items() if v < n * 0.2} - - for i, item in enumerate(items): - if not isinstance(item, dict): - continue - - item_fields = set(item.keys()) - - # Has rare fields that most items don't have - has_rare = bool(item_fields & rare_fields) - if has_rare: - outlier_indices.append(i) - continue - - # 2. Detect rare status/state values - # Find fields that look like status fields (low cardinality, categorical) - status_outliers = _detect_rare_status_values(items, common_fields) - outlier_indices.extend(status_outliers) - - return list(set(outlier_indices)) - - -def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> list[int]: - """Detect items with rare values in status-like fields. - - A status field has low cardinality (few distinct values). - If 95%+ have the same value, items with different values are interesting. - """ - outlier_indices: list[int] = [] - - # Find potential status fields (low cardinality) - for field_name in common_fields: - values = [ - item.get(field_name) for item in items if isinstance(item, dict) and field_name in item - ] - - # Skip if too few values or non-hashable - try: - unique_values = {str(v) for v in values if v is not None} - except Exception: - continue - - # BUG #3 FIX: cardinality cap raised from 10 to 50 so that - # higher-cardinality but Pareto-distributed fields (e.g. - # 60 INFO + 25 WARN + 15 distinct error codes) still get - # their rare values flagged. Above 50 distinct values, the - # field is almost certainly an ID/free-form column, not a - # status enum. - if not (2 <= len(unique_values) <= 50): - continue - - # Count value frequencies - value_counts: dict[str, int] = {} - for v in values: - key = str(v) if v is not None else "__none__" - value_counts[key] = value_counts.get(key, 0) + 1 - - if not value_counts: - continue - - total = len(values) - - # BUG #3 FIX: replace single-dominant check with Pareto top-K. - # Sort frequencies descending; tiebreak by key ascending so the - # outcome is deterministic when multiple values have the same - # frequency. Find the smallest K such that top-K covers >=80% - # of items. - sorted_counts = sorted(value_counts.items(), key=lambda x: (-x[1], x[0])) - threshold = math.ceil(total * 0.8) - cumulative = 0 - top_k_values: set[str] = set() - for value, count in sorted_counts: - cumulative += count - top_k_values.add(value) - if cumulative >= threshold: - break - - # K must be small (<=5) for any value to count as "rare". - # Above this the distribution is too uniform to label a value - # rare. - if len(top_k_values) > 5: - continue - - # Items NOT in top_k_values are outliers. - for i, item in enumerate(items): - if not isinstance(item, dict) or field_name not in item: - continue - item_value = str(item[field_name]) if item[field_name] is not None else "__none__" - if item_value not in top_k_values: - outlier_indices.append(i) - - return outlier_indices - - -# Error keywords for PRESERVATION guarantee (not crushability detection) -# This is for the quality guarantee: "ALL error items are ALWAYS preserved" -# regardless of how common they are. Used in _prioritize_indices(). -# Centralized in error_detection module for consistency across transforms. -_ERROR_KEYWORDS_FOR_PRESERVATION = ERROR_KEYWORDS - - -def _detect_error_items_for_preservation( - items: list[dict], - item_strings: list[str] | None = None, -) -> list[int]: - """Detect items containing error keywords for PRESERVATION guarantee. - - This is NOT for crushability analysis - it's for ensuring ALL error items - are retained during compression. The quality guarantee is that error items - are NEVER dropped, even if errors are common in the dataset. - - Uses keywords because error semantics are well-defined across domains. - - Args: - items: List of items to check. - item_strings: Pre-computed JSON serializations to avoid redundant json.dumps. - """ - error_indices: list[int] = [] - - for i, item in enumerate(items): - if not isinstance(item, dict): - continue - - # Reuse cached serialization if available, otherwise serialize - try: - if item_strings is not None and i < len(item_strings): - item_str = item_strings[i].lower() - else: - item_str = json.dumps(item).lower() - except Exception: - continue - - # Check if any error keyword is present - for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION: - if keyword in item_str: - error_indices.append(i) - break - - return error_indices - - -def _detect_items_by_learned_semantics( - items: list[dict], - field_semantics: dict[str, FieldSemantics], -) -> list[int]: - """Detect items with important values based on learned field semantics. - - This is the TOIN Evolution integration - uses learned field semantic types - to identify items that should be preserved during compression. - - Key insight: Instead of hardcoded patterns, we learn from user behavior - which field values are actually important (e.g., error indicators, rare - status values, identifiers that get queried). - - Args: - items: List of items to analyze. - field_semantics: Learned field semantics from TOIN (field_hash -> FieldSemantics). - - Returns: - List of indices for items containing important values. - """ - if not field_semantics or not items: - return [] - - important_indices: list[int] = [] - - # Build a quick lookup for field_hash -> FieldSemantics - # Pre-filter to fields with sufficient confidence - confident_semantics = { - fh: fs - for fh, fs in field_semantics.items() - if fs.confidence >= 0.3 and fs.inferred_type != "unknown" - } - - if not confident_semantics: - return [] - - # Pre-compute field name hashes to avoid redundant SHA256 per item - _field_hash_cache: dict[str, str] = {} - - for i, item in enumerate(items): - if not isinstance(item, dict): - continue - - for field_name, value in item.items(): - # Hash the field name to match TOIN's format (cached per unique field name) - if field_name not in _field_hash_cache: - _field_hash_cache[field_name] = _hash_field_name(field_name) - field_hash = _field_hash_cache[field_name] - - if field_hash not in confident_semantics: - continue - - field_sem = confident_semantics[field_hash] - - # Hash the value to check importance - if value is None: - value_canonical = "null" - elif isinstance(value, bool): - value_canonical = "true" if value else "false" - elif isinstance(value, int | float): - value_canonical = str(value) - elif isinstance(value, str): - value_canonical = value - elif isinstance(value, list | dict): - try: - value_canonical = json.dumps(value, sort_keys=True, default=str) - except (TypeError, ValueError): - value_canonical = str(value) - else: - value_canonical = str(value) - - value_hash = hashlib.sha256(value_canonical.encode()).hexdigest()[:8] - - # Check if this value is important based on learned semantics - if field_sem.is_value_important(value_hash): - important_indices.append(i) - break # Only need to mark item once - - return important_indices - - -@dataclass -class CrushabilityAnalysis: - """Analysis of whether an array is safe to crush. - - The key insight: if we don't have a reliable SIGNAL to determine - which items are important, we should NOT crush at all. - - Signals include: - - Score/rank fields (search results) - - Error keywords (logs) - - Numeric anomalies (metrics) - - Low uniqueness (repetitive data where sampling is representative) - - High variability + No signal = DON'T CRUSH - """ - - crushable: bool - confidence: float # 0.0 to 1.0 - reason: str - signals_present: list[str] = field(default_factory=list) - signals_absent: list[str] = field(default_factory=list) - - # Detailed metrics - has_id_field: bool = False - id_uniqueness: float = 0.0 - avg_string_uniqueness: float = 0.0 - has_score_field: bool = False - error_item_count: int = 0 - anomaly_count: int = 0 - - -@dataclass -class FieldStats: - """Statistics for a single field across array items.""" - - name: str - field_type: str # "numeric", "string", "boolean", "object", "array", "null" - count: int - unique_count: int - unique_ratio: float - is_constant: bool - constant_value: Any = None - - # Numeric-specific stats - min_val: float | None = None - max_val: float | None = None - mean_val: float | None = None - variance: float | None = None - change_points: list[int] = field(default_factory=list) - - # String-specific stats - avg_length: float | None = None - top_values: list[tuple[str, int]] = field(default_factory=list) - - -@dataclass -class ArrayAnalysis: - """Complete analysis of an array.""" - - item_count: int - field_stats: dict[str, FieldStats] - detected_pattern: str # "time_series", "logs", "search_results", "generic" - recommended_strategy: CompressionStrategy - constant_fields: dict[str, Any] - estimated_reduction: float - crushability: CrushabilityAnalysis | None = None # Whether it's safe to crush - - -@dataclass -class CompressionPlan: - """Plan for how to compress an array.""" - - strategy: CompressionStrategy - keep_indices: list[int] = field(default_factory=list) - constant_fields: dict[str, Any] = field(default_factory=dict) - summary_ranges: list[tuple[int, int, dict]] = field(default_factory=list) - cluster_field: str | None = None - sort_field: str | None = None - keep_count: int = 10 +# ─── Public dataclasses ─────────────────────────────────────────────────── @dataclass class CrushResult: - """Result from SmartCrusher.crush() method. + """Result from `SmartCrusher.crush()`. - Used by ContentRouter when routing JSON arrays to SmartCrusher. + Used by `ContentRouter` when routing JSON arrays to `SmartCrusher`. """ compressed: str @@ -989,590 +60,44 @@ class CrushResult: @dataclass class SmartCrusherConfig: - """Configuration for smart crusher. + """Configuration for SmartCrusher. - SCHEMA-PRESERVING: Output contains only items from the original array. - No wrappers, no generated text, no metadata keys. + SCHEMA-PRESERVING: output contains only items from the original + array. No wrappers, no generated text, no metadata keys. + + Field names + defaults match the Rust `SmartCrusherConfig` byte-for- + byte; the shim copies these straight into the PyO3 constructor. """ enabled: bool = True - min_items_to_analyze: int = 5 # Don't analyze tiny arrays - min_tokens_to_crush: int = 200 # Only crush if > N tokens - variance_threshold: float = 2.0 # Std devs for change point detection - uniqueness_threshold: float = 0.1 # Below this = nearly constant - similarity_threshold: float = 0.8 # For clustering similar strings - max_items_after_crush: int = 15 # Target max items in output + min_items_to_analyze: int = 5 + min_tokens_to_crush: int = 200 + variance_threshold: float = 2.0 + uniqueness_threshold: float = 0.1 + similarity_threshold: float = 0.8 + max_items_after_crush: int = 15 preserve_change_points: bool = True - factor_out_constants: bool = False # Disabled - preserves original schema - include_summaries: bool = False # Disabled - no generated text - - # Feedback loop integration - use_feedback_hints: bool = True # Use learned patterns to adjust compression - - # LOW FIX #21: Make TOIN confidence threshold configurable - # Minimum confidence required to apply TOIN recommendations + factor_out_constants: bool = False + include_summaries: bool = False + use_feedback_hints: bool = True toin_confidence_threshold: float = 0.5 - - # Content deduplication - prevents wasting slots on identical items dedup_identical_items: bool = True + first_fraction: float = 0.3 + last_fraction: float = 0.15 - # Adaptive K boundary allocation (fraction of total K for first/last items) - first_fraction: float = 0.3 # 30% of K from start of array - last_fraction: float = 0.15 # 15% of K from end of array - -class SmartAnalyzer: - """Analyzes JSON arrays to determine optimal compression strategy.""" - - def __init__(self, config: SmartCrusherConfig | None = None): - self.config = config or SmartCrusherConfig() - - def analyze_array(self, items: list[dict]) -> ArrayAnalysis: - """Perform complete statistical analysis of an array.""" - if not items or not isinstance(items[0], dict): - return ArrayAnalysis( - item_count=len(items) if items else 0, - field_stats={}, - detected_pattern="generic", - recommended_strategy=CompressionStrategy.NONE, - constant_fields={}, - estimated_reduction=0.0, - ) - - # Analyze each field - field_stats = {} - all_keys: set[str] = set() - for item in items: - if isinstance(item, dict): - all_keys.update(item.keys()) - - # PARITY FIX (Stage 3c.1): iterate in sorted key order. Python's - # set iteration is non-deterministic across PYTHONHASHSEED, so - # downstream short-circuits (`_select_strategy` "message" lookup, - # `_detect_pattern` first-score-field) become non-deterministic. - # Rust uses BTreeMap which iterates ASCII-sorted; sorting here - # locks both languages to the same iteration order so parity - # fixtures byte-match. - for key in sorted(all_keys): - field_stats[key] = self._analyze_field(key, items) - - # Detect pattern - pattern = self._detect_pattern(field_stats, items) - - # Extract constants - constant_fields = {k: v.constant_value for k, v in field_stats.items() if v.is_constant} - - # CRITICAL: Analyze crushability BEFORE selecting strategy - crushability = self.analyze_crushability(items, field_stats) - - # Select strategy (respects crushability) - strategy = self._select_strategy(field_stats, pattern, len(items), crushability) - - # Estimate reduction (0 if not crushable) - if strategy == CompressionStrategy.SKIP: - reduction = 0.0 - else: - reduction = self._estimate_reduction(field_stats, strategy, len(items)) - - return ArrayAnalysis( - item_count=len(items), - field_stats=field_stats, - detected_pattern=pattern, - recommended_strategy=strategy, - constant_fields=constant_fields, - estimated_reduction=reduction, - crushability=crushability, - ) - - def _analyze_field(self, key: str, items: list[dict]) -> FieldStats: - """Analyze a single field across all items.""" - values = [item.get(key) for item in items if isinstance(item, dict)] - non_null_values = [v for v in values if v is not None] - - if not non_null_values: - return FieldStats( - name=key, - field_type="null", - count=len(values), - unique_count=0, - unique_ratio=0.0, - is_constant=True, - constant_value=None, - ) - - # Determine type from first non-null value - first_val = non_null_values[0] - if isinstance(first_val, bool): - field_type = "boolean" - elif isinstance(first_val, int | float): - field_type = "numeric" - elif isinstance(first_val, str): - field_type = "string" - elif isinstance(first_val, dict): - field_type = "object" - elif isinstance(first_val, list): - field_type = "array" - else: - field_type = "unknown" - - # Compute uniqueness - str_values = [str(v) for v in values] - unique_values = set(str_values) - unique_count = len(unique_values) - unique_ratio = unique_count / len(values) if values else 0 - - # Check if constant - is_constant = unique_count == 1 - constant_value = non_null_values[0] if is_constant else None - - stats = FieldStats( - name=key, - field_type=field_type, - count=len(values), - unique_count=unique_count, - unique_ratio=unique_ratio, - is_constant=is_constant, - constant_value=constant_value, - ) - - # Numeric-specific analysis - if field_type == "numeric": - # Filter out NaN and Infinity which break statistics functions - nums = [v for v in non_null_values if isinstance(v, int | float) and math.isfinite(v)] - if nums: - try: - stats.min_val = min(nums) - stats.max_val = max(nums) - stats.mean_val = statistics.mean(nums) - stats.variance = statistics.variance(nums) if len(nums) > 1 else 0 - stats.change_points = self._detect_change_points(nums) - except (OverflowError, ValueError): - # Extreme values that overflow - skip detailed statistics - stats.min_val = None - stats.max_val = None - stats.mean_val = None - stats.variance = 0 - stats.change_points = [] - - # String-specific analysis - elif field_type == "string": - strs = [v for v in non_null_values if isinstance(v, str)] - if strs: - stats.avg_length = statistics.mean(len(s) for s in strs) - stats.top_values = Counter(strs).most_common(5) - - return stats - - def _detect_change_points(self, values: list[float], window: int = 5) -> list[int]: - """Detect indices where values change significantly.""" - if len(values) < window * 2: - return [] - - change_points = [] - - # Calculate overall statistics - overall_std = statistics.stdev(values) if len(values) > 1 else 0 - if overall_std == 0: - return [] - - threshold = self.config.variance_threshold * overall_std - - # Sliding window comparison - for i in range(window, len(values) - window): - before_mean = statistics.mean(values[i - window : i]) - after_mean = statistics.mean(values[i : i + window]) - - if abs(after_mean - before_mean) > threshold: - change_points.append(i) - - # Deduplicate nearby change points - if change_points: - deduped = [change_points[0]] - for cp in change_points[1:]: - if cp - deduped[-1] > window: - deduped.append(cp) - return deduped - - return [] - - def _detect_pattern(self, field_stats: dict[str, FieldStats], items: list[dict]) -> str: - """Detect the data pattern using STATISTICAL analysis (no hardcoded field names). - - Pattern detection: - - TIME_SERIES: Has a temporal field (detected by value format) + numeric variance - - LOGS: Has a high-cardinality string field + low-cardinality categorical field - - SEARCH_RESULTS: Has a score-like field (bounded numeric, possibly sorted) - - GENERIC: Default - """ - # Check for time series pattern using STRUCTURAL detection - has_timestamp = self._detect_temporal_field(field_stats, items) - - numeric_fields = [k for k, v in field_stats.items() if v.field_type == "numeric"] - has_numeric_with_variance = any( - (field_stats[k].variance is not None and (field_stats[k].variance or 0) > 0) - for k in numeric_fields - ) - - if has_timestamp and has_numeric_with_variance: - return "time_series" - - # Check for logs pattern using STATISTICAL detection - # Logs have: high-cardinality string (message) + low-cardinality categorical (level) - has_message_like = False - has_level_like = False - - for _name, stats in field_stats.items(): - if stats.field_type == "string": - # High-cardinality string = likely message field - if stats.unique_ratio > 0.5 and stats.avg_length and stats.avg_length > 20: - has_message_like = True - # Low-cardinality string = likely level/status field - elif stats.unique_ratio < 0.1 and 2 <= stats.unique_count <= 10: - has_level_like = True - - if has_message_like and has_level_like: - return "logs" - - # Check for search results pattern using STATISTICAL score detection - for _name, stats in field_stats.items(): - is_score, confidence = _detect_score_field_statistically(stats, items) - if is_score and confidence >= 0.5: - return "search_results" - - return "generic" - - def _detect_temporal_field(self, field_stats: dict[str, FieldStats], items: list[dict]) -> bool: - """Detect if any field contains temporal values (dates/timestamps). - - Uses STRUCTURAL detection based on value format, not field names. - """ - # Check string fields for ISO 8601 patterns (module-level compiled) - iso_datetime_pattern = _ISO_DATETIME_PATTERN - iso_date_pattern = _ISO_DATE_PATTERN - - for name, stats in field_stats.items(): - if stats.field_type == "string": - # Sample some values - sample_values = [ - item.get(name) for item in items[:10] if isinstance(item.get(name), str) - ] - if sample_values: - # Check if values look like dates/datetimes - iso_count = sum( - 1 - for v in sample_values - if v is not None - and (iso_datetime_pattern.match(v) or iso_date_pattern.match(v)) - ) - if iso_count / len(sample_values) > 0.5: - return True - - # Check numeric fields for Unix timestamp range - elif stats.field_type == "numeric": - if stats.min_val and stats.max_val: - # Unix timestamps (seconds): 1000000000 to 2000000000 (roughly 2001-2033) - # Unix timestamps (milliseconds): 1000000000000 to 2000000000000 - is_unix_seconds = 1000000000 <= stats.min_val <= 2000000000 - is_unix_millis = 1000000000000 <= stats.min_val <= 2000000000000 - if is_unix_seconds or is_unix_millis: - return True - - return False - - def analyze_crushability( - self, - items: list[dict], - field_stats: dict[str, FieldStats], - ) -> CrushabilityAnalysis: - """Analyze whether it's SAFE to crush this array. - - The key insight: High variability + No importance signal = DON'T CRUSH. - - We use STATISTICAL detection (no hardcoded field names): - 1. ID fields detected by uniqueness + sequential/UUID/entropy patterns - 2. Score fields detected by bounded range + sorted order - 3. Error items detected by structural outliers (rare fields, rare status values) - 4. Numeric anomalies (importance signal) - 5. Low uniqueness (safe to sample) - - Returns: - CrushabilityAnalysis with decision and reasoning. - """ - signals_present: list[str] = [] - signals_absent: list[str] = [] - - # 1. Detect ID field STATISTICALLY (no hardcoded field names) - id_field_name = None - id_uniqueness = 0.0 - id_confidence = 0.0 - for name, stats in field_stats.items(): - values = [item.get(name) for item in items if isinstance(item, dict)] - is_id, confidence = _detect_id_field_statistically(stats, values) - if is_id and confidence > id_confidence: - id_field_name = name - id_uniqueness = stats.unique_ratio - id_confidence = confidence - - has_id_field = id_field_name is not None and id_confidence >= 0.7 - - # 2. Detect score/rank field STATISTICALLY (no hardcoded field names) - has_score_field = False - for name, stats in field_stats.items(): - is_score, confidence = _detect_score_field_statistically(stats, items) - if is_score: - has_score_field = True - signals_present.append(f"score_field:{name}(conf={confidence:.2f})") - break - if not has_score_field: - signals_absent.append("score_field") - - # 3. Detect error items via STRUCTURAL OUTLIERS (no hardcoded keywords) - outlier_indices = _detect_structural_outliers(items) - structural_outlier_count = len(outlier_indices) - - if structural_outlier_count > 0: - signals_present.append(f"structural_outliers:{structural_outlier_count}") - else: - signals_absent.append("structural_outliers") - - # 3b. Also detect errors via keywords in content (for log/message-style data) - # This catches errors that are in the content but not structural outliers - # (e.g., Slack messages where error is in the text field) - error_keyword_indices = _detect_error_items_for_preservation(items) - keyword_error_count = len(error_keyword_indices) - - if keyword_error_count > 0 and structural_outlier_count == 0: - signals_present.append(f"error_keywords:{keyword_error_count}") - - # Combined error count for crushability analysis - error_count = max(structural_outlier_count, keyword_error_count) - - # 4. Count numeric anomalies (importance signal) - anomaly_count = 0 - anomaly_indices: set[int] = set() - for stats in field_stats.values(): - if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance: - std = stats.variance**0.5 - if std > 0: - threshold = self.config.variance_threshold * std - for i, item in enumerate(items): - val = item.get(stats.name) - if isinstance(val, int | float): - if abs(val - stats.mean_val) > threshold: - anomaly_indices.add(i) - - anomaly_count = len(anomaly_indices) - if anomaly_count > 0: - signals_present.append(f"anomalies:{anomaly_count}") - else: - signals_absent.append("anomalies") - - # 5. Compute average string uniqueness (EXCLUDING statistically-detected ID fields) - string_stats = [ - s for s in field_stats.values() if s.field_type == "string" and s.name != id_field_name - ] - avg_string_uniqueness = ( - statistics.mean(s.unique_ratio for s in string_stats) if string_stats else 0.0 - ) - - # Compute uniqueness of non-ID numeric fields - non_id_numeric_stats = [ - s for s in field_stats.values() if s.field_type == "numeric" and s.name != id_field_name - ] - avg_non_id_numeric_uniqueness = ( - statistics.mean(s.unique_ratio for s in non_id_numeric_stats) - if non_id_numeric_stats - else 0.0 - ) - - # Combined uniqueness metric (including ID fields) - max_uniqueness = max(avg_string_uniqueness, id_uniqueness, 0.0) - - # Non-ID content uniqueness (for detecting repetitive content with unique IDs) - non_id_content_uniqueness = max(avg_string_uniqueness, avg_non_id_numeric_uniqueness) - - # 6. Check for change points (importance signal for time series) - has_change_points = any( - stats.change_points for stats in field_stats.values() if stats.field_type == "numeric" - ) - if has_change_points: - signals_present.append("change_points") - - # DECISION LOGIC - has_any_signal = len(signals_present) > 0 - - # Case 0: Repetitive content with unique IDs - # If all non-ID fields are nearly constant, data is safe to sample - # even if there's a unique ID field (e.g., status="success" for all items) - if non_id_content_uniqueness < 0.1 and has_id_field: - signals_present.append("repetitive_content") - return CrushabilityAnalysis( - crushable=True, - confidence=0.85, - reason="repetitive_content_with_ids", - signals_present=signals_present, - signals_absent=signals_absent, - has_id_field=has_id_field, - id_uniqueness=id_uniqueness, - avg_string_uniqueness=avg_string_uniqueness, - has_score_field=has_score_field, - error_item_count=error_count, - anomaly_count=anomaly_count, - ) - - # Case 1: Low uniqueness - safe to sample (data is repetitive) - if max_uniqueness < 0.3: - return CrushabilityAnalysis( - crushable=True, - confidence=0.9, - reason="low_uniqueness_safe_to_sample", - signals_present=signals_present, - signals_absent=signals_absent, - has_id_field=has_id_field, - id_uniqueness=id_uniqueness, - avg_string_uniqueness=avg_string_uniqueness, - has_score_field=has_score_field, - error_item_count=error_count, - anomaly_count=anomaly_count, - ) - - # Case 2: High uniqueness + ID field + NO signal = DON'T CRUSH - # This is the critical case: DB results, file listings, user lists - if has_id_field and max_uniqueness > 0.8 and not has_any_signal: - return CrushabilityAnalysis( - crushable=False, - confidence=0.85, - reason="unique_entities_no_signal", - signals_present=signals_present, - signals_absent=signals_absent, - has_id_field=has_id_field, - id_uniqueness=id_uniqueness, - avg_string_uniqueness=avg_string_uniqueness, - has_score_field=has_score_field, - error_item_count=error_count, - anomaly_count=anomaly_count, - ) - - # Case 3: High uniqueness + has signal = CRUSH using signal - if max_uniqueness > 0.8 and has_any_signal: - return CrushabilityAnalysis( - crushable=True, - confidence=0.7, - reason="unique_entities_with_signal", - signals_present=signals_present, - signals_absent=signals_absent, - has_id_field=has_id_field, - id_uniqueness=id_uniqueness, - avg_string_uniqueness=avg_string_uniqueness, - has_score_field=has_score_field, - error_item_count=error_count, - anomaly_count=anomaly_count, - ) - - # Case 4: Medium uniqueness + no signal = be cautious, don't crush - if not has_any_signal: - return CrushabilityAnalysis( - crushable=False, - confidence=0.6, - reason="medium_uniqueness_no_signal", - signals_present=signals_present, - signals_absent=signals_absent, - has_id_field=has_id_field, - id_uniqueness=id_uniqueness, - avg_string_uniqueness=avg_string_uniqueness, - has_score_field=has_score_field, - error_item_count=error_count, - anomaly_count=anomaly_count, - ) - - # Case 5: Medium uniqueness + has signal = crush with caution - return CrushabilityAnalysis( - crushable=True, - confidence=0.5, - reason="medium_uniqueness_with_signal", - signals_present=signals_present, - signals_absent=signals_absent, - has_id_field=has_id_field, - id_uniqueness=id_uniqueness, - avg_string_uniqueness=avg_string_uniqueness, - has_score_field=has_score_field, - error_item_count=error_count, - anomaly_count=anomaly_count, - ) - - def _select_strategy( - self, - field_stats: dict[str, FieldStats], - pattern: str, - item_count: int, - crushability: CrushabilityAnalysis | None = None, - ) -> CompressionStrategy: - """Select optimal compression strategy based on analysis.""" - if item_count < self.config.min_items_to_analyze: - return CompressionStrategy.NONE - - # CRITICAL: Check crushability first - if crushability is not None and not crushability.crushable: - return CompressionStrategy.SKIP - - if pattern == "time_series": - # Check if there are change points worth preserving - numeric_fields = [v for v in field_stats.values() if v.field_type == "numeric"] - has_change_points = any(f.change_points for f in numeric_fields) - if has_change_points: - return CompressionStrategy.TIME_SERIES - - if pattern == "logs": - # Check if messages are clusterable (low-medium uniqueness) - message_field = next( - (v for k, v in field_stats.items() if "message" in k.lower()), None - ) - if message_field and message_field.unique_ratio < 0.5: - return CompressionStrategy.CLUSTER_SAMPLE - - if pattern == "search_results": - return CompressionStrategy.TOP_N - - # Default: smart sampling - return CompressionStrategy.SMART_SAMPLE - - def _estimate_reduction( - self, field_stats: dict[str, FieldStats], strategy: CompressionStrategy, item_count: int - ) -> float: - """Estimate token reduction ratio.""" - if strategy == CompressionStrategy.NONE: - return 0.0 - - # Count constant fields (will be factored out) - constant_ratio = sum(1 for v in field_stats.values() if v.is_constant) / len(field_stats) - - # Estimate based on strategy - base_reduction = { - CompressionStrategy.TIME_SERIES: 0.7, - CompressionStrategy.CLUSTER_SAMPLE: 0.8, - CompressionStrategy.TOP_N: 0.6, - CompressionStrategy.SMART_SAMPLE: 0.5, - }.get(strategy, 0.3) - - # Adjust for constants - reduction = base_reduction + (constant_ratio * 0.2) - - return min(reduction, 0.95) +# ─── Rust-backed SmartCrusher ───────────────────────────────────────────── class SmartCrusher(Transform): - """ - Intelligent tool output compression using statistical analysis. + """Rust-backed `SmartCrusher` (via PyO3 / `headroom._core`). - Unlike fixed-rule crushing, SmartCrusher: - 1. Analyzes JSON structure and computes field statistics - 2. Detects data patterns (time series, logs, search results) - 3. Identifies constant fields to factor out - 4. Finds change points in numeric data to preserve - 5. Applies optimal compression strategy per data type - 6. Uses RelevanceScorer for semantic matching of user queries - - This results in higher compression with lower information loss. + Same `__init__` and method shapes as the retired Python class — + drop-in replacement. The `crush()` and `_smart_crush_content()` + methods delegate every byte to Rust; `apply()` keeps the + Transform-protocol orchestration in Python (message walking, + digest-marker insertion, token counting) since that's mostly glue + around the per-message compression call. """ name = "smart_crusher" @@ -1580,597 +105,139 @@ class SmartCrusher(Transform): def __init__( self, config: SmartCrusherConfig | None = None, - relevance_config: RelevanceScorerConfig | None = None, - scorer: RelevanceScorer | None = None, + relevance_config: Any = None, + scorer: Any = None, ccr_config: CCRConfig | None = None, ): - self.config = config or SmartCrusherConfig() - self.analyzer = SmartAnalyzer(self.config) + # Hard import — no Python fallback. If the wheel is missing the + # caller must build it (scripts/build_rust_extension.sh) or + # install a prebuilt one. Failing loudly is better than silent + # degradation; see feedback memory `feedback_no_silent_fallbacks.md`. + from headroom._core import ( + SmartCrusher as _RustSmartCrusher, + ) + from headroom._core import ( + SmartCrusherConfig as _RustSmartCrusherConfig, + ) - # CCR (Compress-Cache-Retrieve) configuration - # When no ccr_config provided, default to caching enabled but markers disabled - # This maintains backward compatibility - callers must opt-in to markers + cfg = config or SmartCrusherConfig() + self.config = cfg + + # CCR config is preserved on `self` for callers that read it + # back (`headroom.proxy.server` does), but the Rust port doesn't + # exercise it: Stage 3c.1 keeps CCR marker injection disabled + # because the Rust port has no compression store. When Stage + # 3c.2 lands the CCR port, this wires through to Rust. if ccr_config is None: - self._ccr_config = CCRConfig( - enabled=True, # Still cache for potential retrieval - inject_retrieval_marker=False, # Don't break JSON parsing by default - ) + self._ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False) else: self._ccr_config = ccr_config - self._compression_store: CompressionStore | None = None - # Feedback loop for learning compression patterns - self._feedback: CompressionFeedback | None = None + # `relevance_config` and `scorer` are accepted for source + # compatibility but currently dropped — Stage 3c.1 ships with + # the Rust default `HybridScorer`. Custom scorers re-attach in + # Stage 3c.2 when the relevance crate gains a Python-bridged + # constructor surface. + if relevance_config is not None or scorer is not None: + logger.debug( + "SmartCrusher: relevance_config/scorer args are ignored in " + "Stage 3c.1 (Rust port uses default HybridScorer). They " + "will be wired through in Stage 3c.2." + ) - # CRITICAL FIX: Lock for thread-safe lazy initialization - # Without this, multiple threads could call _get_* methods simultaneously - # and potentially create redundant initialization calls. - self._lazy_init_lock = threading.Lock() - - # Initialize relevance scorer - if scorer is not None: - self._scorer = scorer - else: - rel_config = relevance_config or RelevanceScorerConfig() - # Build kwargs based on tier - BM25 params only apply to bm25 tier - scorer_kwargs = {} - if rel_config.tier == "bm25": - scorer_kwargs = {"k1": rel_config.bm25_k1, "b": rel_config.bm25_b} - elif rel_config.tier == "hybrid": - scorer_kwargs = { - "alpha": rel_config.hybrid_alpha, - "adaptive": rel_config.adaptive_alpha, - } - self._scorer = create_scorer(tier=rel_config.tier, **scorer_kwargs) - # Use threshold from config, or default from RelevanceScorerConfig - rel_cfg = relevance_config or RelevanceScorerConfig() - self._relevance_threshold = rel_cfg.relevance_threshold - - # Initialize AnchorSelector for dynamic position-based preservation - anchor_config = self.config.anchor if hasattr(self.config, "anchor") else AnchorConfig() - self._anchor_selector = AnchorSelector(anchor_config) - - # NOTE: Error detection now uses structural outlier detection (_detect_structural_outliers) - # instead of hardcoded keywords. This scales to any data domain. - - def _map_to_anchor_pattern(self, strategy: CompressionStrategy) -> AnchorDataPattern: - """Map SmartCrusher compression strategy to AnchorSelector data pattern. - - Args: - strategy: The detected compression strategy. - - Returns: - Corresponding AnchorDataPattern for anchor selection. - """ - return { - CompressionStrategy.TIME_SERIES: AnchorDataPattern.TIME_SERIES, - CompressionStrategy.TOP_N: AnchorDataPattern.SEARCH_RESULTS, - CompressionStrategy.CLUSTER_SAMPLE: AnchorDataPattern.LOGS, - CompressionStrategy.SMART_SAMPLE: AnchorDataPattern.GENERIC, - }.get(strategy, AnchorDataPattern.GENERIC) + # Build the Rust crusher with every field from the Python + # config, plus the relevance_threshold default (0.3) — the + # Python dataclass doesn't carry that field; it lives on + # `RelevanceScorerConfig` instead. + self._rust = _RustSmartCrusher( + _RustSmartCrusherConfig( + enabled=cfg.enabled, + min_items_to_analyze=cfg.min_items_to_analyze, + min_tokens_to_crush=cfg.min_tokens_to_crush, + variance_threshold=cfg.variance_threshold, + uniqueness_threshold=cfg.uniqueness_threshold, + similarity_threshold=cfg.similarity_threshold, + max_items_after_crush=cfg.max_items_after_crush, + preserve_change_points=cfg.preserve_change_points, + factor_out_constants=cfg.factor_out_constants, + include_summaries=cfg.include_summaries, + use_feedback_hints=cfg.use_feedback_hints, + toin_confidence_threshold=cfg.toin_confidence_threshold, + dedup_identical_items=cfg.dedup_identical_items, + first_fraction=cfg.first_fraction, + last_fraction=cfg.last_fraction, + relevance_threshold=0.3, + ) + ) def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult: - """Crush content string directly (for use by ContentRouter). + """Crush a single JSON content string. - This is a simplified interface for compressing a single content string, - used by ContentRouter when routing JSON arrays to SmartCrusher. - - Args: - content: JSON string content to compress. - query: Query context for relevance-based compression. - bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer). - - Returns: - CrushResult with compressed content and metadata. + Mirrors the retired Python method. Returns a `CrushResult` + dataclass so call sites that destructure with `asdict()` keep + working. """ - compressed, was_modified, analysis_info = self._smart_crush_content( - content, query_context=query, bias=bias - ) + r = self._rust.crush(content, query, bias) return CrushResult( - compressed=compressed, - original=content, - was_modified=was_modified, - strategy=analysis_info or "passthrough", + compressed=r.compressed, + original=r.original, + was_modified=r.was_modified, + strategy=r.strategy, ) - def _get_compression_store(self) -> CompressionStore: - """Get the compression store for CCR (lazy initialization). - - CRITICAL FIX: Thread-safe double-checked locking pattern. - """ - if self._compression_store is None: - with self._lazy_init_lock: - # Double-check after acquiring lock - if self._compression_store is None: - self._compression_store = get_compression_store( - max_entries=self._ccr_config.store_max_entries, - default_ttl=self._ccr_config.store_ttl_seconds, - ) - return self._compression_store - - def _get_feedback(self) -> CompressionFeedback: - """Get the feedback analyzer (lazy initialization). - - CRITICAL FIX: Thread-safe double-checked locking pattern. - """ - if self._feedback is None: - with self._lazy_init_lock: - if self._feedback is None: - self._feedback = get_compression_feedback() - return self._feedback - - def _get_telemetry(self) -> TelemetryCollector: - """Get the telemetry collector (lazy initialization). - - CRITICAL FIX: Thread-safe double-checked locking pattern. - """ - # Use getattr to avoid hasattr race condition - if getattr(self, "_telemetry", None) is None: - with self._lazy_init_lock: - if getattr(self, "_telemetry", None) is None: - self._telemetry = get_telemetry_collector() - return self._telemetry - - def _get_toin(self) -> ToolIntelligenceNetwork: - """Get the TOIN instance (lazy initialization). - - CRITICAL FIX: Thread-safe double-checked locking pattern. - """ - # Use getattr to avoid hasattr race condition - if getattr(self, "_toin", None) is None: - with self._lazy_init_lock: - if getattr(self, "_toin", None) is None: - self._toin = get_toin() - return self._toin - - def _record_telemetry( + def _smart_crush_content( self, - items: list[dict], - result: list, - analysis: ArrayAnalysis, - plan: CompressionPlan, + content: str, + query_context: str = "", tool_name: str | None = None, - ) -> None: - """Record compression telemetry for the data flywheel. + bias: float = 1.0, + ) -> tuple[str, bool, str]: + """Apply smart crushing; return `(crushed, was_modified, info)`. - This collects anonymized statistics about compression patterns to - enable cross-user learning and improve compression over time. - - Privacy guarantees: - - No actual data values are stored - - Tool names can be hashed - - Only structural patterns are captured + Mirrors the retired Python method's tuple shape. `tool_name` is + accepted for API compatibility and currently ignored — Stage + 3c.1 has no per-tool TOIN learning hook. """ - try: - telemetry = self._get_telemetry() + crushed, was_modified, info = self._rust.smart_crush_content(content, query_context, bias) + return crushed, was_modified, info - # Calculate what was kept - kept_first_n = sum(1 for i in plan.keep_indices if i < 3) - kept_last_n = sum(1 for i in plan.keep_indices if i >= len(items) - 2) + def _extract_context_from_messages(self, messages: list[dict[str, Any]]) -> str: + """Build a query string from the last 5 user messages + recent + assistant tool-call arguments. Used by `apply()` to derive the + relevance context per-request. - # Count error items in result - error_indices = set(_detect_error_items_for_preservation(items)) - kept_errors = sum(1 for i in plan.keep_indices if i in error_indices) - - # Count anomalies (approximate from change points) - anomaly_count = 0 - for stats in analysis.field_stats.values(): - if stats.change_points: - anomaly_count += len(stats.change_points) - kept_anomalies = min(anomaly_count, len(plan.keep_indices)) - - # Crushability info - crushability_score = None - crushability_reason = None - if analysis.crushability: - crushability_score = analysis.crushability.confidence - crushability_reason = analysis.crushability.reason - - # Record the event - telemetry.record_compression( - items=items[:100], # Sample for structure analysis - original_count=len(items), - compressed_count=len(result), - original_tokens=0, # Not available here - compressed_tokens=0, # Not available here - strategy=analysis.recommended_strategy.value, - tool_name=tool_name, - strategy_reason=analysis.detected_pattern, - crushability_score=crushability_score, - crushability_reason=crushability_reason, - kept_first_n=kept_first_n, - kept_last_n=kept_last_n, - kept_errors=kept_errors, - kept_anomalies=kept_anomalies, - kept_by_relevance=0, # Would need to track separately - kept_by_score=0, # Would need to track separately - ) - except Exception: - # Telemetry should never break compression - pass - - def _deduplicate_indices_by_content( - self, - keep_indices: set[int], - items: list[dict], - ) -> set[int]: - """Deduplicate indices by content hash, preferring lower indices. - - When multiple indices contain identical content, only the lowest index - is kept. This maximizes information density in the compressed output. - - Enterprise considerations: - - Thread-safe: No shared state modified - - Performance: O(n) single pass with hash map - - Memory: O(n) for hash storage (16-char hashes) - - Fault-tolerant: Serialization errors preserve the item - - Args: - keep_indices: Set of indices to deduplicate. - items: The items array for content lookup. - - Returns: - Deduplicated set of indices (may be smaller than input). + Pure Python because it walks the message envelope, not the + compressed payload. The retired implementation lived inline on + `SmartCrusher`; preserved here unchanged. """ - if not keep_indices: - return keep_indices + context_parts: list[str] = [] + user_message_count = 0 - # Track first occurrence of each content hash - # Using dict[hash -> lowest_index] for O(1) lookup - seen_hashes: dict[str, int] = {} - duplicates_removed = 0 - - # Process in sorted order to ensure deterministic "lowest index wins" - for idx in sorted(keep_indices): - # Bounds check - if idx < 0 or idx >= len(items): - continue - - item = items[idx] - - # Compute content hash - try: - if isinstance(item, dict): - # Canonical JSON serialization for consistent hashing - content = json.dumps(item, sort_keys=True, default=str) - else: - # Non-dict items: use string representation - content = str(item) - item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324 - except (TypeError, ValueError, RecursionError) as e: - # Serialization failed - keep the item (fail-safe) - logger.debug("Dedup hash failed for item at index %d: %s. Keeping item.", idx, e) - # Use index as unique "hash" to ensure item is kept - item_hash = f"__idx_{idx}__" - - # First occurrence wins - if item_hash not in seen_hashes: - seen_hashes[item_hash] = idx - else: - duplicates_removed += 1 - - # Log deduplication stats for observability - if duplicates_removed > 0: - logger.debug( - "Content deduplication removed %d duplicate items from %d candidates " - "(%.1f%% reduction). Unique items: %d", - duplicates_removed, - len(keep_indices), - 100 * duplicates_removed / len(keep_indices), - len(seen_hashes), - ) - - return set(seen_hashes.values()) - - def _fill_remaining_slots( - self, - keep_indices: set[int], - items: list[dict], - n: int, - effective_max: int, - ) -> set[int]: - """Fill remaining slots with unique items when under budget. - - When deduplication reduces keep_indices below effective_max, this method - fills the remaining slots with diverse items from the array. It: - 1. Computes content hashes for already-kept items - 2. Scans array for items with unique content not yet kept - 3. Distributes new items evenly across the array for coverage - - Enterprise considerations: - - Thread-safe: No shared state modified - - Performance: O(n) for hash computation, O(n) for filling - - Memory: O(k) where k = len(keep_indices) for hash storage - - Deterministic: Same input always produces same output - - Args: - keep_indices: Current set of indices to keep (already deduplicated). - items: The full items array. - n: Total number of items. - effective_max: Target maximum items. - - Returns: - Updated set of indices, filled up to effective_max with unique items. - """ - remaining_slots = effective_max - len(keep_indices) - if remaining_slots <= 0: - return keep_indices - - # Build set of content hashes for items we're already keeping - seen_hashes: set[str] = set() - for idx in keep_indices: - if 0 <= idx < n: - item = items[idx] - try: - if isinstance(item, dict): - content = json.dumps(item, sort_keys=True, default=str) - else: - content = str(item) - seen_hashes.add(hashlib.md5(content.encode()).hexdigest()[:16]) # nosec B324 - except (TypeError, ValueError, RecursionError): - pass # Skip hash computation failures - - # Find candidate indices not in keep_indices - candidates = [i for i in range(n) if i not in keep_indices] - if not candidates: - return keep_indices - - # Distribute selection evenly across the array for coverage - # Use stride-based sampling to avoid clustering - result = keep_indices.copy() - added = 0 - - # Calculate step size for even distribution - step = max(1, len(candidates) // (remaining_slots + 1)) - - # First pass: evenly distributed unique items - for start_offset in range(step): - if added >= remaining_slots: - break - for i in range(start_offset, len(candidates), step): - if added >= remaining_slots: - break - idx = candidates[i] - item = items[idx] - - # Check if this item's content is unique - try: - if isinstance(item, dict): - content = json.dumps(item, sort_keys=True, default=str) - else: - content = str(item) - item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324 - except (TypeError, ValueError, RecursionError): - # Hash failure - use index as unique hash (fail-safe) - item_hash = f"__idx_{idx}__" - - if item_hash not in seen_hashes: - result.add(idx) - seen_hashes.add(item_hash) - added += 1 - - if added > 0: - logger.debug( - "Filled %d remaining slots (from %d to %d items) after deduplication", - added, - len(keep_indices), - len(result), - ) - - return result - - def _prioritize_indices( - self, - keep_indices: set[int], - items: list[dict], - n: int, - analysis: ArrayAnalysis | None = None, - max_items: int | None = None, - field_semantics: dict[str, FieldSemantics] | None = None, - ) -> set[int]: - """Prioritize indices when we exceed max_items, ALWAYS keeping critical items. - - Priority order: - 1. ALL error items (non-negotiable) - items with error keywords - 2. ALL structural outliers (non-negotiable) - items with rare fields/status values - 3. ALL numeric anomalies (non-negotiable) - e.g., unusual values like 999999 - 4. ALL items with important values (learned) - TOIN field semantics - 5. First 3 items (context) - 6. Last 2 items (context) - 7. Other important items by index order - - Uses BOTH keyword detection (for preservation guarantee) AND statistical detection, - PLUS learned field semantics from TOIN for zero-latency signal detection. - - HIGH FIX: Note that this function may return MORE items than effective_max - when critical items (errors, outliers, anomalies) exceed the limit. This is - intentional to preserve the quality guarantee. A warning is logged when this - happens to help diagnose cases where compression is less effective than expected. - - Args: - keep_indices: Initial set of indices to keep. - items: The items being compressed. - n: Total number of items. - analysis: Optional analysis results for anomaly detection. - max_items: Thread-safe max items limit (defaults to config value). - field_semantics: Optional learned field semantics from TOIN. - - Returns: - Set of indices to keep (may exceed max_items if critical items require it). - """ - # Use provided max_items or fall back to config - effective_max = max_items if max_items is not None else self.config.max_items_after_crush - - # === ENTERPRISE FIX: Content-based deduplication === - # Multiple preservation mechanisms (anchors, anomalies, outliers, etc.) can add - # the same item multiple times by index. More critically, different INDICES can - # contain IDENTICAL content (e.g., 10 identical status messages at indices 0-9). - # Without deduplication, we waste slots on redundant information. - # - # This deduplication runs FIRST, before any other logic, to ensure: - # 1. Budget calculations work with unique items only - # 2. Critical item detection doesn't double-count - # 3. Final output maximizes information density - # - # Performance: O(n) where n = len(keep_indices), single pass with hash map - # Memory: O(n) for hash storage, hashes are 16 chars each - if self.config.dedup_identical_items: - keep_indices = self._deduplicate_indices_by_content(keep_indices, items) - - # === ENTERPRISE FIX: Fill up to max_items when under budget === - # After deduplication, we may have fewer unique items than max_items. - # Instead of returning a sparse result, fill remaining slots with - # diverse items from the array. This maximizes information density. - if len(keep_indices) < effective_max and len(keep_indices) < n: - keep_indices = self._fill_remaining_slots(keep_indices, items, n, effective_max) - - if len(keep_indices) <= effective_max: - return keep_indices - - # Use provided field_semantics or fall back to thread-local (set by _crush_array) - effective_field_semantics = field_semantics or getattr( - getattr(self, "_thread_local", None), "field_semantics", None - ) - - # Identify error items using KEYWORD detection (preservation guarantee) - # This ensures ALL error items are kept, regardless of frequency - error_indices = set(_detect_error_items_for_preservation(items)) - - # Identify structural outlier indices using STATISTICAL detection - # (items with rare fields or rare status values) - outlier_indices = set(_detect_structural_outliers(items)) - - # Identify numeric anomalies (MUST keep ALL of them) - anomaly_indices = set() - if analysis and analysis.field_stats: - for field_name, stats in analysis.field_stats.items(): - if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance: - std = stats.variance**0.5 - if std > 0: - threshold = self.config.variance_threshold * std - for i, item in enumerate(items): - val = item.get(field_name) - if isinstance(val, int | float): - if abs(val - stats.mean_val) > threshold: - anomaly_indices.add(i) - - # === TOIN Evolution: Identify items with important values (learned) === - # Uses learned field semantics for zero-latency signal detection - learned_important_indices: set[int] = set() - if effective_field_semantics: - learned_important_indices = set( - _detect_items_by_learned_semantics(items, effective_field_semantics) - ) - - # Start with all critical items (these are non-negotiable) - # Error items are ALWAYS preserved (quality guarantee) - prioritized = error_indices | outlier_indices | anomaly_indices | learned_important_indices - - # HIGH FIX: Log warning if critical items alone exceed the limit - # This helps diagnose why compression may be less effective than expected - critical_count = len(prioritized) - if critical_count > effective_max: - logger.warning( - "Critical items (%d) exceed max_items (%d): errors=%d outliers=%d anomalies=%d learned=%d. " - "Quality guarantee takes precedence - keeping all critical items.", - critical_count, - effective_max, - len(error_indices), - len(outlier_indices), - len(anomaly_indices), - len(learned_important_indices), - ) - - # Add first/last items if we have room - remaining_slots = effective_max - len(prioritized) - if remaining_slots > 0: - # First 3 items - for i in range(min(3, n)): - if i not in prioritized and remaining_slots > 0: - prioritized.add(i) - remaining_slots -= 1 - # Last 2 items - for i in range(max(0, n - 2), n): - if i not in prioritized and remaining_slots > 0: - prioritized.add(i) - remaining_slots -= 1 - - # Fill remaining slots with other important indices (by index order) - if remaining_slots > 0: - other_indices = sorted(keep_indices - prioritized) - for i in other_indices: - if remaining_slots <= 0: - break - prioritized.add(i) - remaining_slots -= 1 - - return prioritized - - def should_apply( - self, - messages: list[dict[str, Any]], - tokenizer: Tokenizer, - **kwargs: Any, - ) -> bool: - """Check if any tool messages would benefit from smart crushing.""" - if not self.config.enabled: - return False - - for msg in messages: - # OpenAI style: role="tool" - if msg.get("role") == "tool": - content = msg.get("content", "") + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content") if isinstance(content, str): - tokens = tokenizer.count_text(content) - if tokens > self.config.min_tokens_to_crush: - # Check if it's JSON with arrays - parsed, success = safe_json_loads(content) - if success and self._has_crushable_arrays(parsed): - return True + context_parts.append(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + if text: + context_parts.append(text) - # Anthropic style: role="user" with tool_result content blocks - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_result": - tool_content = block.get("content", "") - if isinstance(tool_content, str): - tokens = tokenizer.count_text(tool_content) - if tokens > self.config.min_tokens_to_crush: - parsed, success = safe_json_loads(tool_content) - if success and self._has_crushable_arrays(parsed): - return True + user_message_count += 1 + if user_message_count >= 5: + break - return False + if msg.get("role") == "assistant" and msg.get("tool_calls"): + for tc in msg.get("tool_calls", []): + if isinstance(tc, dict): + func = tc.get("function", {}) + args = func.get("arguments", "") + if isinstance(args, str) and args: + context_parts.append(args) - def _has_crushable_arrays(self, data: Any, depth: int = 0) -> bool: - """Check if data contains arrays large enough to crush. - - Accepts arrays of ANY homogeneous type (dicts, strings, numbers, etc.) - as well as mixed-type arrays. Bool-only arrays are excluded (not useful - to compress). - """ - if depth > 5: - return False - - if isinstance(data, list): - if len(data) >= self.config.min_items_to_analyze: - arr_type = _classify_array(data) - if arr_type not in (ArrayType.EMPTY, ArrayType.BOOL_ARRAY): - return True - for item in data[:10]: # Check first few items - if self._has_crushable_arrays(item, depth + 1): - return True - - elif isinstance(data, dict): - # Large objects with many keys are themselves crushable - if len(data) >= self.config.min_items_to_analyze: - return True - for value in data.values(): - if self._has_crushable_arrays(value, depth + 1): - return True - - return False + return " ".join(context_parts) def apply( self, @@ -2178,76 +245,68 @@ class SmartCrusher(Transform): tokenizer: Tokenizer, **kwargs: Any, ) -> TransformResult: - """Apply smart crushing to messages.""" + """Transform-protocol entry point. Walks every tool/tool_result + message, applies SmartCrusher to large enough payloads, and + replaces the message content with `\\n`. + + Pure orchestration — the per-message compression delegates to + Rust via `_smart_crush_content`. + """ tokens_before = tokenizer.count_messages(messages) result_messages = deep_copy_messages(messages) transforms_applied: list[str] = [] markers_inserted: list[str] = [] warnings: list[str] = [] - # Extract query context from recent user messages for relevance scoring query_context = self._extract_context_from_messages(result_messages) - crushed_count = 0 frozen_message_count = kwargs.get("frozen_message_count", 0) for msg_idx, msg in enumerate(result_messages): - # Skip frozen messages (in provider's prefix cache) if msg_idx < frozen_message_count: continue - # OpenAI style + # OpenAI-style: top-level role=tool with string content. if msg.get("role") == "tool": content = msg.get("content", "") - if not isinstance(content, str): - continue + if isinstance(content, str): + tokens = tokenizer.count_text(content) + if tokens > self.config.min_tokens_to_crush: + crushed, was_modified, info = self._smart_crush_content( + content, query_context + ) + if was_modified: + marker = create_tool_digest_marker(compute_short_hash(content)) + msg["content"] = crushed + "\n" + marker + crushed_count += 1 + markers_inserted.append(marker) + if info: + transforms_applied.append(f"smart:{info}") - tokens = tokenizer.count_text(content) - if tokens <= self.config.min_tokens_to_crush: - continue - - crushed, was_modified, analysis_info = self._smart_crush_content( - content, query_context - ) - - if was_modified: - original_hash = compute_short_hash(content) - marker = create_tool_digest_marker(original_hash) - msg["content"] = crushed + "\n" + marker - crushed_count += 1 - markers_inserted.append(marker) - if analysis_info: - transforms_applied.append(f"smart:{analysis_info}") - - # Anthropic style + # Anthropic-style: content is a list of blocks; each tool_result + # block has a string content field of its own. content = msg.get("content") if isinstance(content, list): for i, block in enumerate(content): - if not isinstance(block, dict): + if not isinstance(block, dict) or block.get("type") != "tool_result": continue - if block.get("type") != "tool_result": - continue - tool_content = block.get("content", "") if not isinstance(tool_content, str): continue - tokens = tokenizer.count_text(tool_content) if tokens <= self.config.min_tokens_to_crush: continue - crushed, was_modified, analysis_info = self._smart_crush_content( + crushed, was_modified, info = self._smart_crush_content( tool_content, query_context ) - if was_modified: - original_hash = compute_short_hash(tool_content) - marker = create_tool_digest_marker(original_hash) + marker = create_tool_digest_marker(compute_short_hash(tool_content)) content[i]["content"] = crushed + "\n" + marker crushed_count += 1 markers_inserted.append(marker) - if analysis_info: - transforms_applied.append(f"smart:{analysis_info}") + if info: + transforms_applied.append(f"smart:{info}") if crushed_count > 0: transforms_applied.insert(0, f"smart_crush:{crushed_count}") @@ -2263,1455 +322,8 @@ class SmartCrusher(Transform): warnings=warnings, ) - def _extract_context_from_messages(self, messages: list[dict[str, Any]]) -> str: - """Extract query context from recent messages for relevance scoring. - Builds a context string from: - - Recent user messages (what the user is asking about) - - Recent tool call arguments (what data was requested) - - This context is used by RelevanceScorer to determine which items - to preserve during crushing. - - Args: - messages: Full message list. - - Returns: - Context string for relevance scoring. - """ - context_parts: list[str] = [] - - # Look at last 5 user messages (most relevant to recent tool calls) - user_message_count = 0 - for msg in reversed(messages): - if msg.get("role") == "user": - content = msg.get("content") - if isinstance(content, str): - context_parts.append(content) - elif isinstance(content, list): - # Anthropic style - extract from text blocks - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text = block.get("text", "") - if text: - context_parts.append(text) - - user_message_count += 1 - if user_message_count >= 5: - break - - # Also check assistant tool_calls for function arguments - if msg.get("role") == "assistant" and msg.get("tool_calls"): - for tc in msg.get("tool_calls", []): - if isinstance(tc, dict): - func = tc.get("function", {}) - args = func.get("arguments", "") - if isinstance(args, str) and args: - context_parts.append(args) - - return " ".join(context_parts) - - def _smart_crush_content( - self, - content: str, - query_context: str = "", - tool_name: str | None = None, - bias: float = 1.0, - ) -> tuple[str, bool, str]: - """ - Apply smart crushing to content. - - Handles both JSON (existing SmartCrusher logic) and plain text content - (search results, logs, generic text) using specialized compressors. - - Args: - content: Content to crush (JSON or plain text). - query_context: Context string from user messages for relevance scoring. - tool_name: Name of the tool that produced this output. - bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer). - - Returns: - Tuple of (crushed_content, was_modified, analysis_info). - """ - parsed, success = safe_json_loads(content) - if not success: - # Not JSON - pass through unchanged - # Text compression utilities (SearchCompressor, LogCompressor, TextCompressor) - # are available as standalone tools for applications to use explicitly - return content, False, "" - - # Recursively process and crush arrays - crushed, info, ccr_markers = self._process_value( - parsed, query_context=query_context, tool_name=tool_name, bias=bias - ) - - result = safe_json_dumps(crushed, indent=None) - was_modified = result != content.strip() - - # CCR: Inject retrieval markers if compression happened and CCR is enabled - if was_modified and ccr_markers and self._ccr_config.inject_retrieval_marker: - for marker_data in ccr_markers: - if len(marker_data) == 4: - ccr_hash, original_count, compressed_count, dropped_summary = marker_data - else: - ccr_hash, original_count, compressed_count = marker_data - dropped_summary = "" - summary_str = f" Omitted: {dropped_summary}." if dropped_summary else "" - # Escape { } in summary to prevent .format() errors - safe_summary = summary_str.replace("{", "{{").replace("}", "}}") - ttl_seconds = getattr(self._ccr_config, "store_ttl_seconds", 300) - marker = self._ccr_config.marker_template.format( - original_count=original_count, - compressed_count=compressed_count, - hash=ccr_hash, - summary=safe_summary, - ttl_minutes=max(1, ttl_seconds // 60), - ) - result += marker - - return result, was_modified, info - - # Maximum recursion depth for nested JSON processing. - # Prevents RecursionError on adversarial/deeply-nested input. - _MAX_PROCESS_DEPTH = 50 - - def _process_value( - self, - value: Any, - depth: int = 0, - query_context: str = "", - tool_name: str | None = None, - bias: float = 1.0, - ) -> tuple[Any, str, list[tuple[str, int, int]]]: - """Recursively process a value, crushing arrays where appropriate. - - Returns: - Tuple of (processed_value, info_string, ccr_markers). - ccr_markers is a list of (hash, original_count, compressed_count, summary) tuples. - """ - # Guard against deeply nested JSON causing RecursionError - if depth >= self._MAX_PROCESS_DEPTH: - return value, "", [] - - info_parts = [] - ccr_markers: list[tuple] = [] - - if isinstance(value, list): - if len(value) >= self.config.min_items_to_analyze: - arr_type = _classify_array(value) - - if arr_type == ArrayType.DICT_ARRAY: - # Existing path — dict arrays (battle-tested, unchanged) - crushed, strategy, ccr_hash, dropped_summary = self._crush_array( - value, query_context, tool_name, bias=bias - ) - info_parts.append(f"{strategy}({len(value)}->{len(crushed)})") - if ccr_hash: - ccr_markers.append((ccr_hash, len(value), len(crushed), dropped_summary)) - return crushed, ",".join(info_parts), ccr_markers - - elif arr_type == ArrayType.STRING_ARRAY: - crushed, strategy = self._crush_string_array(value, bias=bias) - info_parts.append(f"{strategy}({len(value)}->{len(crushed)})") - return crushed, ",".join(info_parts), ccr_markers - - elif arr_type == ArrayType.NUMBER_ARRAY: - crushed, strategy = self._crush_number_array(value, bias=bias) - if isinstance(crushed, list): - info_parts.append(f"{strategy}({len(value)}->{len(crushed)})") - else: - info_parts.append(f"{strategy}({len(value)}->summary)") - return crushed, ",".join(info_parts), ccr_markers - - elif arr_type == ArrayType.MIXED_ARRAY: - crushed, strategy = self._crush_mixed_array( - value, query_context, tool_name, bias=bias - ) - info_parts.append(f"{strategy}({len(value)}->{len(crushed)})") - return crushed, ",".join(info_parts), ccr_markers - - # NESTED_ARRAY, BOOL_ARRAY, EMPTY — fall through to recursive - - # Not crushable or below threshold — process items recursively - processed = [] - for item in value: - p_item, p_info, p_markers = self._process_value( - item, depth + 1, query_context, tool_name, bias=bias - ) - processed.append(p_item) - if p_info: - info_parts.append(p_info) - ccr_markers.extend(p_markers) - return processed, ",".join(info_parts), ccr_markers - - elif isinstance(value, dict): - # First: recurse into values to compress nested arrays - processed_dict: dict[str, Any] = {} - for k, v in value.items(): - p_val, p_info, p_markers = self._process_value( - v, depth + 1, query_context, tool_name, bias=bias - ) - processed_dict[k] = p_val - if p_info: - info_parts.append(p_info) - ccr_markers.extend(p_markers) - - # Second: if the object itself has many keys, compress at key level - if len(processed_dict) >= self.config.min_items_to_analyze: - crushed_dict, strategy = self._crush_object(processed_dict, bias=bias) - if strategy != "object:passthrough": - info_parts.append(strategy) - return crushed_dict, ",".join(info_parts), ccr_markers - - return processed_dict, ",".join(info_parts), ccr_markers - - else: - return value, "", [] - - def _crush_array( - self, - items: list[dict], - query_context: str = "", - tool_name: str | None = None, - bias: float = 1.0, - ) -> tuple[list, str, str | None, str]: - """Crush an array using statistical analysis and relevance scoring. - - IMPORTANT: If crushability analysis determines it's not safe to crush - (high variability + no importance signal), returns original array unchanged. - - TOIN-aware: Consults the Tool Output Intelligence Network for cross-user - learned patterns. High retrieval rate across all users → compress less. - - Feedback-aware: Uses learned patterns to adjust compression aggressiveness. - High retrieval rate for a tool → compress less aggressively. - - Args: - items: List of dict items to compress. - query_context: Context string from user messages for relevance scoring. - tool_name: Name of the tool that produced this output. - bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer). - - Returns: - Tuple of (crushed_items, strategy_info, ccr_hash, dropped_summary). - ccr_hash is the hash for retrieval if CCR is enabled, None otherwise. - dropped_summary is a categorical summary of what was dropped. - """ - # BOUNDARY CHECK: Use adaptive sizing instead of hardcoded limit - # compute_optimal_k handles trivial cases (n <= 8 → keep all) - from .adaptive_sizer import compute_optimal_k - - item_strings = [json.dumps(item, default=str) for item in items] - adaptive_k = compute_optimal_k( - item_strings, - bias=bias, - min_k=3, - max_k=self.config.max_items_after_crush if self.config.max_items_after_crush else None, - ) - - if len(items) <= adaptive_k: - # All items kept (high diversity or small array). - # Instead of passing through unchanged, try to compress the TEXT - # WITHIN each item — reduce token count without losing any item. - compressed_items = _compress_text_within_items(items, query_context) - if compressed_items is not items: - return compressed_items, "compress_within:diversity", None, "" - return items, "none:adaptive_at_limit", None, "" - - # Get feedback hints if enabled - # THREAD-SAFETY: Use a local effective_max_items instead of mutating shared config - effective_max_items = adaptive_k - hints_applied = False - toin_hint_applied = False - - # Create ToolSignature for TOIN lookup - tool_signature = ToolSignature.from_items(items) - - # TOIN: Get cross-user learned recommendations - toin = self._get_toin() - toin_hint = toin.get_recommendation(tool_signature, query_context) - - # Log TOIN hint details - logger.debug( - "TOIN hint: source=%s, confidence=%.2f, skip=%s, max_items=%d", - toin_hint.source, - toin_hint.confidence, - toin_hint.skip_compression, - toin_hint.max_items, - ) - - if toin_hint.skip_compression: - return items, f"skip:toin({toin_hint.reason})", None, "" - - # Apply TOIN recommendations if from network or local learning - toin_preserve_fields: list[str] = [] - toin_recommended_strategy: str | None = None - toin_compression_level: str | None = None - # LOW FIX #21: Use configurable threshold instead of hardcoded 0.5 - if ( - toin_hint.source in ("network", "local") - and toin_hint.confidence >= self.config.toin_confidence_threshold - ): - # TOIN recommendations take precedence over local feedback - effective_max_items = toin_hint.max_items - toin_preserve_fields = toin_hint.preserve_fields # Fields to never remove - toin_hint_applied = True - # Store strategy and compression level for later use - if toin_hint.recommended_strategy != "default": - toin_recommended_strategy = toin_hint.recommended_strategy - if toin_hint.compression_level != "moderate": - toin_compression_level = toin_hint.compression_level - # Log that TOIN hint was applied - logger.debug( - "TOIN hint applied: max_items=%d, strategy=%s, compression_level=%s", - effective_max_items, - toin_recommended_strategy or "default", - toin_compression_level or "moderate", - ) - elif toin_hint.source in ("network", "local"): - # Hint available but confidence too low - logger.debug( - "TOIN hint not applied: confidence %.2f < threshold %.2f", - toin_hint.confidence, - self.config.toin_confidence_threshold, - ) - - # === TOIN Evolution: Extract field semantics for signal detection === - # Store in thread-local storage for use in _prioritize_indices. - # This enables learned signal detection without changing all method signatures - # while remaining thread-safe (no cross-thread contamination). - if not hasattr(self, "_thread_local"): - self._thread_local = threading.local() - self._thread_local.field_semantics = ( - toin_hint.field_semantics if toin_hint.field_semantics else None - ) - - # Local feedback hints (if TOIN didn't apply) - if not toin_hint_applied and self.config.use_feedback_hints and tool_name: - feedback = self._get_feedback() - hints = feedback.get_compression_hints(tool_name) - - # Check if hints recommend skipping compression - if hints.skip_compression: - return items, f"skip:feedback({hints.reason})", None, "" - - # Adjust max_items based on feedback - if hints.suggested_items is not None: - effective_max_items = hints.suggested_items - hints_applied = True - - # Use preserve_fields from local feedback (hash them for TOIN compatibility) - # Note: CompressionFeedback stores actual field names, but _plan methods - # expect SHA256[:8] hashes for privacy-preserving comparison - if hints.preserve_fields: - toin_preserve_fields = [_hash_field_name(field) for field in hints.preserve_fields] - - # Use recommended_strategy from local feedback if not already set by TOIN - if hints.recommended_strategy and not toin_recommended_strategy: - toin_recommended_strategy = hints.recommended_strategy - - try: - # Analyze the array (includes crushability check) - analysis = self.analyzer.analyze_array(items) - - # CRITICAL: If not crushable, return original array unchanged - if analysis.recommended_strategy == CompressionStrategy.SKIP: - reason = "" - if analysis.crushability: - reason = f"skip:{analysis.crushability.reason}" - return items, reason, None, "" - - # Apply TOIN strategy recommendation if available - # TOIN learns which strategies work best from cross-user patterns - if toin_recommended_strategy: - try: - toin_strategy = CompressionStrategy(toin_recommended_strategy) - # Only override if TOIN suggests a valid non-SKIP strategy - if toin_strategy != CompressionStrategy.SKIP: - analysis.recommended_strategy = toin_strategy - except ValueError: - pass # Invalid strategy name, keep analyzer's choice - - # Apply TOIN compression level to adjust effective_max_items - if toin_compression_level: - if toin_compression_level == "none": - # Don't compress - return original - return items, "skip:toin_level_none", None, "" - elif toin_compression_level == "conservative": - # Be conservative - keep more items - effective_max_items = max(effective_max_items, min(50, len(items) // 2)) - elif toin_compression_level == "aggressive": - # Be aggressive - keep fewer items - effective_max_items = min(effective_max_items, 15) - - # Create compression plan with relevance scoring - # Pass TOIN preserve_fields so items with those fields get priority - # Pass effective_max_items for thread-safe compression - # Pass item_strings to avoid redundant json.dumps across plan methods - plan = self._create_plan( - analysis, - items, - query_context, - preserve_fields=toin_preserve_fields or None, - effective_max_items=effective_max_items, - item_strings=item_strings, - ) - - # Execute compression - result = self._execute_plan(plan, items, analysis) - - # CCR: Store original content for retrieval if enabled - ccr_hash = None - if ( - self._ccr_config.enabled - and len(items) >= self._ccr_config.min_items_to_cache - and len(result) < len(items) # Only cache if compression actually happened - ): - store = self._get_compression_store() - # Reuse cached item_strings to avoid re-serializing - original_json = "[" + ", ".join(item_strings) + "]" - compressed_json = json.dumps(result, default=str) - - ccr_hash = store.store( - original=original_json, - compressed=compressed_json, - original_item_count=len(items), - compressed_item_count=len(result), - tool_name=tool_name, - query_context=query_context, - # CRITICAL: Pass the tool_signature_hash so retrieval events - # can be correlated with compression events in TOIN - tool_signature_hash=tool_signature.structure_hash, - compression_strategy=analysis.recommended_strategy.value, - ) - - # Record compression event for feedback loop - if self.config.use_feedback_hints and tool_name: - feedback = self._get_feedback() - feedback.record_compression( - tool_name=tool_name, - original_count=len(items), - compressed_count=len(result), - strategy=analysis.recommended_strategy.value, - tool_signature_hash=tool_signature.structure_hash, - ) - - # Record telemetry for data flywheel - self._record_telemetry( - items=items, - result=result, - analysis=analysis, - plan=plan, - tool_name=tool_name, - ) - - # TOIN: Record compression event for cross-user learning - try: - # Calculate token counts (approximate) - reuse cached item_strings - original_tokens = sum(len(s) for s in item_strings) // 4 - compressed_tokens = len(json.dumps(result, default=str)) // 4 - - toin.record_compression( - tool_signature=tool_signature, - original_count=len(items), - compressed_count=len(result), - original_tokens=original_tokens, - compressed_tokens=compressed_tokens, - strategy=analysis.recommended_strategy.value, - query_context=query_context, - items=items, # Pass items for field-level semantic learning - ) - except Exception: - # TOIN should never break compression - pass - - strategy_info = analysis.recommended_strategy.value - if toin_hint_applied: - toin_parts = [f"items={toin_hint.max_items}", f"conf={toin_hint.confidence:.2f}"] - if toin_recommended_strategy: - toin_parts.append(f"strategy={toin_recommended_strategy}") - if toin_compression_level and toin_compression_level != "moderate": - toin_parts.append(f"level={toin_compression_level}") - strategy_info += f"(toin:{','.join(toin_parts)})" - elif hints_applied: - strategy_info += f"(feedback:{effective_max_items})" - - # Generate categorical summary of dropped items (use indices, not identity) - from .compression_summary import summarize_dropped_items - - dropped_summary = summarize_dropped_items( - items, - result, - kept_indices=set(plan.keep_indices), - ) - - # Clean up temporary instance variable - if hasattr(self, "_thread_local"): - self._thread_local.field_semantics = None - return result, strategy_info, ccr_hash, dropped_summary - - except Exception: - # Clean up temporary instance variable - if hasattr(self, "_thread_local"): - self._thread_local.field_semantics = None - # Re-raise any exceptions (removed finally block since we no longer mutate config) - raise - - # ================================================================= - # Universal JSON type handlers (string, number, mixed arrays) - # ================================================================= - - def _compute_k_split( - self, - items: list, - bias: float = 1.0, - item_strings: list[str] | None = None, - ) -> tuple[int, int, int, int]: - """Compute adaptive K split into first/last/importance slots. - - Uses the existing Kneedle-based adaptive_sizer for K_total, then - splits according to configurable first_fraction / last_fraction. - - Args: - items: List of items (used as fallback for serialization). - bias: Compression bias multiplier. - item_strings: Pre-computed JSON serializations to avoid redundant json.dumps. - - Returns: - (k_total, k_first, k_last, k_importance) - """ - from .adaptive_sizer import compute_optimal_k - - if item_strings is None: - item_strings = [json.dumps(item, default=str) for item in items] - k_total = compute_optimal_k( - item_strings, - bias=bias, - min_k=3, - max_k=self.config.max_items_after_crush or None, - ) - # BUG #4 FIX: clamp k_first and k_last so their sum never exceeds - # k_total. Without the clamp, k_total=1 produces k_first=k_last=1 - # (both `max(1, round(0))`), violating max_items_after_crush. - # No-op for k_total >= 2 (the common path). - k_first = max(1, round(k_total * self.config.first_fraction)) - k_first = min(k_first, k_total) - k_last = max(1, round(k_total * self.config.last_fraction)) - k_last = min(k_last, max(0, k_total - k_first)) - k_importance = max(0, k_total - k_first - k_last) - return k_total, k_first, k_last, k_importance - - def _crush_string_array( - self, - items: list[str], - bias: float = 1.0, - ) -> tuple[list[str], str]: - """Crush an array of strings using dedup + adaptive sampling. - - Strategy: - 1. Compute adaptive K via Kneedle algorithm - 2. Always keep: error-containing strings, first K, last K - 3. Deduplicate exact matches - 4. Fill remaining budget with diverse samples (stride-based) - - Returns: - (crushed_items, strategy_string) - """ - n = len(items) - if n <= 8: - return items, "string:passthrough" - - k_total, k_first, k_last, k_importance = self._compute_k_split(items, bias) - - # Mandatory: error-containing strings (never dropped) - error_indices: set[int] = set() - for i, s in enumerate(items): - s_lower = s.lower() - for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION: - if keyword in s_lower: - error_indices.add(i) - break - - # Mandatory: strings with abnormal length (anomalies) - lengths = [len(s) for s in items] - if len(lengths) > 1: - mean_len = statistics.mean(lengths) - std_len = statistics.stdev(lengths) - anomaly_indices = { - i - for i, length in enumerate(lengths) - if std_len > 0 and abs(length - mean_len) > self.config.variance_threshold * std_len - } - else: - anomaly_indices = set[int]() - - # Boundary: first K, last K - first_indices = set(range(min(k_first, n))) - last_indices = set(range(max(0, n - k_last), n)) - - # Combine mandatory + boundary - keep_indices = error_indices | anomaly_indices | first_indices | last_indices - - # Dedup: among remaining candidates, skip exact duplicates - seen_strings: set[str] = set() - dedup_count = 0 - for i in sorted(keep_indices): - seen_strings.add(items[i]) - - # Fill remaining budget with diverse stride-based samples - remaining_budget = max(0, k_total - len(keep_indices)) - if remaining_budget > 0: - stride = max(1, (n - 1) // (remaining_budget + 1)) - for i in range(0, n, stride): - if len(keep_indices) >= k_total + len(error_indices) + len(anomaly_indices): - break - if i not in keep_indices: - if items[i] not in seen_strings: - keep_indices.add(i) - seen_strings.add(items[i]) - else: - dedup_count += 1 - - # Build output in original order - result = [items[i] for i in sorted(keep_indices)] - - strategy = f"string:adaptive({n}->{len(result)}" - if dedup_count: - strategy += f",dedup={dedup_count}" - if error_indices: - strategy += f",errors={len(error_indices)}" - strategy += ")" - - return result, strategy - - def _crush_number_array( - self, - items: list[int | float], - bias: float = 1.0, - ) -> tuple[list, str]: - """Crush an array of numbers using statistical summary + outlier preservation. - - Strategy: - 1. Compute descriptive statistics (min, max, mean, median, stddev, percentiles) - 2. Detect outliers (> variance_threshold σ from mean) - 3. Detect change points (sudden shifts in running mean) - 4. Keep: first K, last K, all outliers, change points - 5. Return kept values with a prepended stats summary string - - Returns: - (crushed_items, strategy_string) where crushed_items is a list - starting with a summary string followed by representative values. - """ - n = len(items) - if n <= 8: - return items, "number:passthrough" - - # Filter out non-finite values for statistics - finite = [x for x in items if isinstance(x, int | float) and math.isfinite(x)] - if not finite: - return items, "number:no_finite" - - k_total, k_first, k_last, k_importance = self._compute_k_split(items, bias) - - # Statistics - mean_val = statistics.mean(finite) - median_val = statistics.median(finite) - std_val = statistics.stdev(finite) if len(finite) > 1 else 0.0 - sorted_finite = sorted(finite) - # BUG #1 FIX: replace integer-division indexing (off-by-one - # for len < 8) with proper linear-interpolation percentile. - # `_percentile_linear` matches numpy's "linear" method exactly - # — index = q * (n - 1), interpolate between floor and ceil. - p25 = _percentile_linear(sorted_finite, 0.25) - p75 = _percentile_linear(sorted_finite, 0.75) - - # Outliers (> variance_threshold σ from mean) - outlier_indices: set[int] = set() - if std_val > 0: - for i, val in enumerate(items): - if isinstance(val, int | float) and math.isfinite(val): - if abs(val - mean_val) > self.config.variance_threshold * std_val: - outlier_indices.add(i) - - # Change points (detect sudden shifts using running difference) - change_indices: set[int] = set() - if self.config.preserve_change_points and n > 10: - window = 5 - for i in range(window, n - window): - left = [ - items[j] - for j in range(i - window, i) - if isinstance(items[j], int | float) and math.isfinite(items[j]) - ] - right = [ - items[j] - for j in range(i, i + window) - if isinstance(items[j], int | float) and math.isfinite(items[j]) - ] - if left and right: - left_mean = statistics.mean(left) - right_mean = statistics.mean(right) - if ( - std_val > 0 - and abs(right_mean - left_mean) > self.config.variance_threshold * std_val - ): - change_indices.add(i) - - # Boundary: first K, last K - first_indices = set(range(min(k_first, n))) - last_indices = set(range(max(0, n - k_last), n)) - - # Combine all - keep_indices = outlier_indices | change_indices | first_indices | last_indices - - # Fill remaining budget with stride-based samples - remaining_budget = max(0, k_total - len(keep_indices)) - if remaining_budget > 0: - stride = max(1, (n - 1) // (remaining_budget + 1)) - for i in range(0, n, stride): - if len(keep_indices) >= k_total + len(outlier_indices): - break - if i not in keep_indices: - keep_indices.add(i) - - # Build output: kept values only (schema-preserving — no generated text) - kept_values = [items[i] for i in sorted(keep_indices)] - - # Encode statistics into the strategy string (not the array itself) - strategy = ( - f"number:adaptive({n}->{len(kept_values)}" - f",min={min(finite)},max={max(finite)}" - f",mean={mean_val:.4g},median={median_val:.4g}" - f",stddev={std_val:.4g},p25={p25:.4g},p75={p75:.4g}" - ) - if outlier_indices: - strategy += f",outliers={len(outlier_indices)}" - if change_indices: - strategy += f",change_points={len(change_indices)}" - strategy += ")" - - return kept_values, strategy - - def _crush_mixed_array( - self, - items: list, - query_context: str = "", - tool_name: str | None = None, - bias: float = 1.0, - ) -> tuple[list, str]: - """Crush a mixed-type array by grouping items by type and compressing each group. - - Strategy: - 1. Group items by type (dict, str, number, list, None, bool) - 2. For each group with >= min_items_to_analyze items: compress with appropriate handler - 3. For small groups: keep all items - 4. Reassemble in original order - - Returns: - (crushed_items, strategy_string) - """ - n = len(items) - if n <= 8: - return items, "mixed:passthrough" - - # Group items by type, tracking original indices - groups: dict[str, list[tuple[int, Any]]] = {} - for i, item in enumerate(items): - if isinstance(item, dict): - key = "dict" - elif isinstance(item, str): - key = "str" - elif isinstance(item, bool): - key = "bool" - elif isinstance(item, int | float): - key = "number" - elif isinstance(item, list): - key = "list" - elif item is None: - key = "none" - else: - key = "other" - groups.setdefault(key, []).append((i, item)) - - # Compress each group independently - keep_indices: set[int] = set() - strategy_parts: list[str] = [] - - for type_key, group_items in groups.items(): - indices = [idx for idx, _ in group_items] - values = [val for _, val in group_items] - - if len(values) < self.config.min_items_to_analyze: - # Small group — keep all - keep_indices.update(indices) - continue - - if type_key == "dict": - # Use existing dict array crusher - crushed, strategy, _, _ = self._crush_array( - values, query_context, tool_name, bias=bias - ) - crushed_set = {json.dumps(c, sort_keys=True, default=str) for c in crushed} - for idx, val in group_items: - if json.dumps(val, sort_keys=True, default=str) in crushed_set: - keep_indices.add(idx) - strategy_parts.append(f"dict:{len(values)}->{len(crushed)}") - - elif type_key == "str": - crushed, strategy = self._crush_string_array(values, bias=bias) - crushed_set = set(crushed) - for idx, val in group_items: - if val in crushed_set: - keep_indices.add(idx) - strategy_parts.append(f"str:{len(values)}->{len(crushed)}") - - elif type_key == "number": - # For numbers in mixed arrays, just do adaptive sampling (no summary prefix) - k_total, k_first, k_last, _ = self._compute_k_split(values, bias) - first_idx = set(indices[:k_first]) - last_idx = set(indices[-k_last:]) - keep_indices.update(first_idx | last_idx) - # Outliers - finite = [v for v in values if isinstance(v, int | float) and math.isfinite(v)] - if len(finite) > 1: - mean_v = statistics.mean(finite) - std_v = statistics.stdev(finite) - if std_v > 0: - for idx, val in group_items: - if isinstance(val, int | float) and math.isfinite(val): - if abs(val - mean_v) > self.config.variance_threshold * std_v: - keep_indices.add(idx) - strategy_parts.append(f"num:{len(values)}") - - else: - # list, bool, none, other — keep all - keep_indices.update(indices) - - # Reassemble in original order - result = [items[i] for i in sorted(keep_indices)] - - strategy = f"mixed:adaptive({n}->{len(result)},{','.join(strategy_parts)})" - return result, strategy - - def _crush_object( - self, - obj: dict[str, Any], - bias: float = 1.0, - ) -> tuple[dict[str, Any], str]: - """Crush a large JSON object by selecting the most informative keys. - - Treats key-value pairs as items and applies adaptive K to select which - keys to retain. Preserves schema — each kept key-value pair is exact - from the original. - - Strategy: - 1. Classify each value by size (tokens) and importance - 2. Always keep: keys with small values (cheap), keys with error content - 3. Compute adaptive K on key-value representations - 4. Fill remaining budget with diverse keys (stride-based) - - Returns: - (compressed_object, strategy_string) - """ - n = len(obj) - if n <= 8: - return obj, "object:passthrough" - - # Estimate tokens per key-value pair - kv_tokens: list[tuple[str, int]] = [] - total_tokens = 0 - for key, val in obj.items(): - val_str = json.dumps(val, default=str) - tokens = len(val_str) // 4 + len(key) // 4 + 2 # rough estimate - kv_tokens.append((key, tokens)) - total_tokens += tokens - - # If already small enough, passthrough - if total_tokens < self.config.min_tokens_to_crush: - return obj, "object:passthrough" - - # Compute adaptive K on key-value string representations - keys = list(obj.keys()) - kv_strings = [f"{k}: {json.dumps(obj[k], default=str)}" for k in keys] - - from .adaptive_sizer import compute_optimal_k - - k_total = compute_optimal_k( - kv_strings, - bias=bias, - min_k=3, - max_k=self.config.max_items_after_crush or None, - ) - - if k_total >= n: - return obj, "object:passthrough" - - # Classify keys by importance - keep_keys: set[str] = set() - - # Always keep: keys with error-containing values - for key, val in obj.items(): - val_str = json.dumps(val, default=str).lower() - for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION: - if keyword in val_str: - keep_keys.add(key) - break - - # Always keep: keys with small values (cheap to keep) - small_threshold = 50 # chars - for key, tokens in kv_tokens: - if tokens <= small_threshold // 4: - keep_keys.add(key) - - # Boundary: first K and last K keys - k_first = max(1, round(k_total * self.config.first_fraction)) - k_last = max(1, round(k_total * self.config.last_fraction)) - for key in keys[:k_first]: - keep_keys.add(key) - for key in keys[-k_last:]: - keep_keys.add(key) - - # Fill remaining budget with stride-based diverse sampling - remaining = max(0, k_total - len(keep_keys)) - if remaining > 0: - stride = max(1, (n - 1) // (remaining + 1)) - for i in range(0, n, stride): - if len(keep_keys) >= k_total + len( - [ - k - for k in keep_keys - if any( - kw in json.dumps(obj[k], default=str).lower() - for kw in _ERROR_KEYWORDS_FOR_PRESERVATION - ) - ] - ): - break - keep_keys.add(keys[i]) - - # Build output preserving original key order - result = {k: obj[k] for k in keys if k in keep_keys} - - strategy = f"object:adaptive({n}->{len(result)} keys)" - return result, strategy - - def _create_plan( - self, - analysis: ArrayAnalysis, - items: list[dict], - query_context: str = "", - preserve_fields: list[str] | None = None, - effective_max_items: int | None = None, - item_strings: list[str] | None = None, - ) -> CompressionPlan: - """Create a detailed compression plan using relevance scoring. - - Args: - analysis: The array analysis results. - items: The items to compress. - query_context: Context string from user messages for relevance scoring. - preserve_fields: TOIN-learned fields that users commonly retrieve. - Items with values in these fields get higher priority. - item_strings: Pre-computed JSON serializations to avoid redundant json.dumps. - effective_max_items: Thread-safe max items limit (defaults to config value). - """ - # Use provided effective_max_items or fall back to config - max_items = ( - effective_max_items - if effective_max_items is not None - else self.config.max_items_after_crush - ) - - plan = CompressionPlan( - strategy=analysis.recommended_strategy, - constant_fields=analysis.constant_fields if self.config.factor_out_constants else {}, - ) - - # Handle SKIP - keep all items (shouldn't normally reach here) - if analysis.recommended_strategy == CompressionStrategy.SKIP: - plan.keep_indices = list(range(len(items))) - return plan - - if analysis.recommended_strategy == CompressionStrategy.TIME_SERIES: - plan = self._plan_time_series( - analysis, - items, - plan, - query_context, - preserve_fields, - max_items, - item_strings=item_strings, - ) - - elif analysis.recommended_strategy == CompressionStrategy.CLUSTER_SAMPLE: - plan = self._plan_cluster_sample( - analysis, - items, - plan, - query_context, - preserve_fields, - max_items, - item_strings=item_strings, - ) - - elif analysis.recommended_strategy == CompressionStrategy.TOP_N: - plan = self._plan_top_n( - analysis, - items, - plan, - query_context, - preserve_fields, - max_items, - item_strings=item_strings, - ) - - else: # SMART_SAMPLE or NONE - plan = self._plan_smart_sample( - analysis, - items, - plan, - query_context, - preserve_fields, - max_items, - item_strings=item_strings, - ) - - return plan - - def _plan_time_series( - self, - analysis: ArrayAnalysis, - items: list[dict], - plan: CompressionPlan, - query_context: str = "", - preserve_fields: list[str] | None = None, - max_items: int | None = None, - item_strings: list[str] | None = None, - ) -> CompressionPlan: - """Plan compression for time series data. - - Keeps items around change points (anomalies) plus first/last items. - Uses STATISTICAL outlier detection for important items. - Uses RelevanceScorer for semantic matching of user queries. - - Args: - preserve_fields: TOIN-learned fields that users commonly retrieve. - Items where query_context matches these field values get priority. - max_items: Thread-safe max items limit (defaults to config value). - """ - # Use provided max_items or fall back to config - effective_max = max_items if max_items is not None else self.config.max_items_after_crush - n = len(items) - keep_indices = set() - - # 1. Dynamic anchor selection (replaces static first 3 + last 2) - anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.TIME_SERIES) - anchor_indices = self._anchor_selector.select_anchors( - items=items, - max_items=effective_max, - pattern=anchor_pattern, - query=query_context or None, - ) - keep_indices.update(anchor_indices) - - # 2. Items around change points from numeric fields - for stats in analysis.field_stats.values(): - if stats.change_points: - for cp in stats.change_points: - # Keep a window around each change point - for offset in range(-2, 3): - idx = cp + offset - if 0 <= idx < n: - keep_indices.add(idx) - - # 3. Structural outlier items (STATISTICAL detection - no hardcoded keywords) - outlier_indices = _detect_structural_outliers(items) - keep_indices.update(outlier_indices) - - # 3b. Error items via KEYWORD detection (PRESERVATION GUARANTEE) - # This is critical - errors must ALWAYS be preserved regardless of structure - error_indices = _detect_error_items_for_preservation(items) - keep_indices.update(error_indices) - - # 4. Items matching query anchors (DETERMINISTIC exact match) - # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names) - if query_context: - anchors = extract_query_anchors(query_context) - for i, item in enumerate(items): - if item_matches_anchors(item, anchors): - keep_indices.add(i) - - # 5. Items with high relevance to query context (PROBABILISTIC semantic match) - if query_context: - # Reuse pre-computed item_strings if available - item_strs = ( - item_strings - if item_strings is not None - else [json.dumps(item, default=str) for item in items] - ) - scores = self._scorer.score_batch(item_strs, query_context) - for i, score in enumerate(scores): - if score.score >= self._relevance_threshold: - keep_indices.add(i) - - # 5b. TOIN preserve_fields: boost items where query matches these fields - # Note: preserve_fields are SHA256[:8] hashes, use helper to match - if preserve_fields and query_context: - for i, item in enumerate(items): - if _item_has_preserve_field_match(item, preserve_fields, query_context): - keep_indices.add(i) - - # Limit to effective_max while ALWAYS preserving outliers and anomalies - keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max) - - plan.keep_indices = sorted(keep_indices) - return plan - - def _plan_cluster_sample( - self, - analysis: ArrayAnalysis, - items: list[dict], - plan: CompressionPlan, - query_context: str = "", - preserve_fields: list[str] | None = None, - max_items: int | None = None, - item_strings: list[str] | None = None, - ) -> CompressionPlan: - """Plan compression for clusterable data (like logs). - - Uses clustering plus STATISTICAL outlier detection. - Uses RelevanceScorer for semantic matching of user queries. - - Args: - preserve_fields: TOIN-learned fields that users commonly retrieve. - Items where query_context matches these field values get priority. - max_items: Thread-safe max items limit (defaults to config value). - """ - # Use provided max_items or fall back to config - effective_max = max_items if max_items is not None else self.config.max_items_after_crush - n = len(items) - keep_indices = set() - - # 1. Dynamic anchor selection (replaces static first 3 + last 2) - anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.CLUSTER_SAMPLE) - anchor_indices = self._anchor_selector.select_anchors( - items=items, - max_items=effective_max, - pattern=anchor_pattern, - query=query_context or None, - ) - keep_indices.update(anchor_indices) - - # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords) - outlier_indices = _detect_structural_outliers(items) - keep_indices.update(outlier_indices) - - # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE) - # This is critical - errors must ALWAYS be preserved regardless of structure - error_indices = _detect_error_items_for_preservation(items) - keep_indices.update(error_indices) - - # 3. Cluster by message-like field and keep representatives - # Find a high-cardinality string field (likely message field) - message_field = None - max_uniqueness = 0.0 - for name, stats in analysis.field_stats.items(): - if stats.field_type == "string" and stats.unique_ratio > max_uniqueness: - # Prefer fields with moderate to high uniqueness (message-like) - if stats.unique_ratio > 0.3: - message_field = name - max_uniqueness = stats.unique_ratio - - if message_field: - plan.cluster_field = message_field - - # Simple clustering: group by first 50 chars of message - clusters: dict[str, list[int]] = {} - for i, item in enumerate(items): - msg = str(item.get(message_field, ""))[:50] - msg_hash = hashlib.md5(msg.encode()).hexdigest()[:8] # nosec B324 - if msg_hash not in clusters: - clusters[msg_hash] = [] - clusters[msg_hash].append(i) - - # Keep 1-2 representatives from each cluster - for indices in clusters.values(): - for idx in indices[:2]: - keep_indices.add(idx) - - # 4. Items matching query anchors (DETERMINISTIC exact match) - # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names) - if query_context: - anchors = extract_query_anchors(query_context) - for i, item in enumerate(items): - if item_matches_anchors(item, anchors): - keep_indices.add(i) - - # 5. Items with high relevance to query context (PROBABILISTIC semantic match) - if query_context: - # Reuse pre-computed item_strings if available - item_strs = ( - item_strings - if item_strings is not None - else [json.dumps(item, default=str) for item in items] - ) - scores = self._scorer.score_batch(item_strs, query_context) - for i, score in enumerate(scores): - if score.score >= self._relevance_threshold: - keep_indices.add(i) - - # 5b. TOIN preserve_fields: boost items where query matches these fields - # Note: preserve_fields are SHA256[:8] hashes, use helper to match - if preserve_fields and query_context: - for i, item in enumerate(items): - if _item_has_preserve_field_match(item, preserve_fields, query_context): - keep_indices.add(i) - - # Limit total while ALWAYS preserving outliers and anomalies - keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max) - - plan.keep_indices = sorted(keep_indices) - return plan - - def _plan_top_n( - self, - analysis: ArrayAnalysis, - items: list[dict], - plan: CompressionPlan, - query_context: str = "", - preserve_fields: list[str] | None = None, - max_items: int | None = None, - item_strings: list[str] | None = None, - ) -> CompressionPlan: - """Plan compression for scored/ranked data. - - For data with a score/relevance field, that field IS the primary relevance - signal. Our internal relevance scoring is SECONDARY - it's used to find - potential "needle" items that the original scoring might have missed. - - Strategy: - 1. Keep top N by score (the original system's relevance ranking) - 2. Add structural outliers (errors, anomalies) - 3. Add high-confidence relevance matches (needles the user is looking for) - - Args: - preserve_fields: TOIN-learned fields that users commonly retrieve. - Items where query_context matches these field values get priority. - max_items: Thread-safe max items limit (defaults to config value). - """ - # Use provided max_items or fall back to config - effective_max = max_items if max_items is not None else self.config.max_items_after_crush - - # Find score field using STATISTICAL detection (no hardcoded field names) - score_field = None - max_confidence = 0.0 - for name, stats in analysis.field_stats.items(): - is_score, confidence = _detect_score_field_statistically(stats, items) - if is_score and confidence > max_confidence: - score_field = name - max_confidence = confidence - - if not score_field: - return self._plan_smart_sample( - analysis, - items, - plan, - query_context, - preserve_fields, - effective_max, - item_strings=item_strings, - ) - - plan.sort_field = score_field - keep_indices = set() - - # 1. TOP N by score FIRST (the primary relevance signal) - # The original system's score field is the authoritative ranking - scored_items = [(i, item.get(score_field, 0)) for i, item in enumerate(items)] - scored_items.sort(key=lambda x: x[1], reverse=True) - - # Reserve slots for outliers - top_count = max(0, effective_max - 3) - for idx, _ in scored_items[:top_count]: - keep_indices.add(idx) - - # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords) - outlier_indices = _detect_structural_outliers(items) - keep_indices.update(outlier_indices) - - # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE) - # This is critical - errors must ALWAYS be preserved regardless of structure - error_indices = _detect_error_items_for_preservation(items) - keep_indices.update(error_indices) - - # 3. Items matching query anchors (DETERMINISTIC exact match) - ADDITIVE - # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names) - # These are ALWAYS preserved since they represent explicit user intent - if query_context: - anchors = extract_query_anchors(query_context) - for i, item in enumerate(items): - if i not in keep_indices and item_matches_anchors(item, anchors): - keep_indices.add(i) - - # 4. HIGH-CONFIDENCE relevance matches (potential needles) - ADDITIVE only - # Only add items that are NOT already in top N but match the query strongly - # Use a higher threshold (0.5) since the score field already captures relevance - if query_context: - # Reuse pre-computed item_strings if available - item_strs = ( - item_strings - if item_strings is not None - else [json.dumps(item, default=str) for item in items] - ) - scores = self._scorer.score_batch(item_strs, query_context) - # Higher threshold and limit count to avoid adding everything - high_threshold = max(0.5, self._relevance_threshold * 2) - added_count = 0 - max_relevance_adds = 3 # Limit additional relevance matches - for i, score in enumerate(scores): - if i not in keep_indices and score.score >= high_threshold: - keep_indices.add(i) - added_count += 1 - if added_count >= max_relevance_adds: - break - - # 4b. TOIN preserve_fields: boost items where query matches these fields - # Note: preserve_fields are SHA256[:8] hashes, use helper to match - if preserve_fields and query_context: - for i, item in enumerate(items): - if i not in keep_indices: # Only add if not already kept - if _item_has_preserve_field_match(item, preserve_fields, query_context): - keep_indices.add(i) - - plan.keep_count = len(keep_indices) - plan.keep_indices = sorted(keep_indices) - return plan - - def _plan_smart_sample( - self, - analysis: ArrayAnalysis, - items: list[dict], - plan: CompressionPlan, - query_context: str = "", - preserve_fields: list[str] | None = None, - max_items: int | None = None, - item_strings: list[str] | None = None, - ) -> CompressionPlan: - """Plan smart statistical sampling using STATISTICAL detection. - - Always keeps: - - Dynamic anchor positions (based on data pattern and query context) - - Structural outliers (items with rare fields or rare status values) - - Anomalous numeric items (> 2 std from mean) - - Items around change points - - Items with high relevance to query context (via RelevanceScorer) - - Uses STATISTICAL detection instead of hardcoded keywords. - - Args: - preserve_fields: TOIN-learned fields that users commonly retrieve. - Items where query_context matches these field values get priority. - max_items: Thread-safe max items limit (defaults to config value). - """ - # Use provided max_items or fall back to config - effective_max = max_items if max_items is not None else self.config.max_items_after_crush - - n = len(items) - keep_indices = set() - - # 1. Dynamic anchor selection (replaces static first 3 + last 2) - anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.SMART_SAMPLE) - anchor_indices = self._anchor_selector.select_anchors( - items=items, - max_items=effective_max, - pattern=anchor_pattern, - query=query_context or None, - ) - keep_indices.update(anchor_indices) - - # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords) - outlier_indices = _detect_structural_outliers(items) - keep_indices.update(outlier_indices) - - # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE) - # This is critical - errors must ALWAYS be preserved regardless of structure - error_indices = _detect_error_items_for_preservation(items) - keep_indices.update(error_indices) - - # 3. Anomalous numeric items (> 2 std from mean) - for name, stats in analysis.field_stats.items(): - if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance: - std = stats.variance**0.5 - if std > 0: - threshold = self.config.variance_threshold * std - for i, item in enumerate(items): - val = item.get(name) - if isinstance(val, int | float): - if abs(val - stats.mean_val) > threshold: - keep_indices.add(i) - - # 4. Items around change points (if detected) - if self.config.preserve_change_points: - for stats in analysis.field_stats.values(): - if stats.change_points: - for cp in stats.change_points: - # Keep items around change point - for offset in range(-1, 2): - idx = cp + offset - if 0 <= idx < n: - keep_indices.add(idx) - - # 5. Items matching query anchors (DETERMINISTIC exact match) - # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names) - if query_context: - anchors = extract_query_anchors(query_context) - for i, item in enumerate(items): - if item_matches_anchors(item, anchors): - keep_indices.add(i) - - # 6. Items with high relevance to query context (PROBABILISTIC semantic match) - if query_context: - # Reuse pre-computed item_strings if available - item_strs = ( - item_strings - if item_strings is not None - else [json.dumps(item, default=str) for item in items] - ) - scores = self._scorer.score_batch(item_strs, query_context) - for i, score in enumerate(scores): - if score.score >= self._relevance_threshold: - keep_indices.add(i) - - # 6b. TOIN preserve_fields: boost items where query matches these fields - # Note: preserve_fields are SHA256[:8] hashes, use helper to match - if preserve_fields and query_context: - for i, item in enumerate(items): - if _item_has_preserve_field_match(item, preserve_fields, query_context): - keep_indices.add(i) - - # Limit to effective_max while ALWAYS preserving outliers and anomalies - keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max) - - plan.keep_indices = sorted(keep_indices) - return plan - - def _execute_plan( - self, plan: CompressionPlan, items: list[dict], analysis: ArrayAnalysis - ) -> list: - """Execute a compression plan and return crushed array. - - SCHEMA-PRESERVING: Returns only items from the original array. - No wrappers, no generated text, no metadata keys. - """ - result = [] - - # Return only the kept items, preserving original schema - for idx in sorted(plan.keep_indices): - if 0 <= idx < len(items): - # Copy item unchanged - no modifications to schema - result.append(items[idx].copy()) - - return result +# ─── Convenience function ───────────────────────────────────────────────── def smart_crush_tool_output( @@ -3719,32 +331,12 @@ def smart_crush_tool_output( config: SmartCrusherConfig | None = None, ccr_config: CCRConfig | None = None, ) -> tuple[str, bool, str]: + """Compress a single tool output. Returns `(crushed, was_modified, info)`. + + Convenience wrapper that builds a one-shot `SmartCrusher` per call. + `ccr_config` is accepted for source compatibility but currently + not wired through (Stage 3c.1 keeps CCR marker injection + disabled — Rust port has no compression store). """ - Convenience function to smart-crush a single tool output. - - NOTE: CCR markers are DISABLED by default in this convenience function - to maintain backward compatibility (output remains valid JSON). - To enable CCR markers, pass a CCRConfig with inject_retrieval_marker=True. - - Args: - content: The tool output content (JSON string). - config: Optional SmartCrusher configuration. - ccr_config: Optional CCR (Compress-Cache-Retrieve) configuration. - By default, CCR is enabled (caching) but markers are disabled. - - Returns: - Tuple of (crushed_content, was_modified, analysis_info). - """ - cfg = config or SmartCrusherConfig() - - # Default: CCR enabled for caching, but markers disabled for clean JSON output - if ccr_config is None: - ccr_cfg = CCRConfig( - enabled=True, # Still cache for retrieval - inject_retrieval_marker=False, # Don't break JSON output - ) - else: - ccr_cfg = ccr_config - - crusher = SmartCrusher(cfg, ccr_config=ccr_cfg) + crusher = SmartCrusher(config=config, ccr_config=ccr_config) return crusher._smart_crush_content(content) diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index 6a4d3e680..c8636c084 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -447,11 +447,7 @@ class TestQueryAnchorExtraction: """If user asks for 'Alice', item with Alice should be preserved.""" import json - from headroom.transforms.smart_crusher import ( - SmartCrusher, - SmartCrusherConfig, - extract_query_anchors, - ) + from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig # User is searching for 'Alice' messages = [ @@ -478,38 +474,31 @@ class TestQueryAnchorExtraction: }, ] - # Verify anchor extraction works - anchors = extract_query_anchors("Find the user named 'Alice' in the system.") - assert "alice" in anchors - - # Verify crushing preserves Alice + # End-to-end behavior: the relevance scorer (HybridScorer in + # the Rust port — BM25 + embedding) should pick up "Alice" + # from the user message and preserve the matching tool item + # even though it sits at index 50. config = SmartCrusherConfig( enabled=True, min_items_to_analyze=5, min_tokens_to_crush=100, - max_items_after_crush=10, # Should normally drop Alice at index 50 + max_items_after_crush=10, ) crusher = SmartCrusher(config) tokenizer = get_tokenizer() result = crusher.apply(messages, tokenizer) - # Find the crushed tool output tool_msg = next(m for m in result.messages if m.get("role") == "tool") crushed_content = tool_msg["content"] - # Alice should be preserved even though she's at index 50 assert "Alice" in crushed_content def test_preserves_needle_by_uuid(self): """If user asks for a UUID, item with that UUID should be preserved.""" import json - from headroom.transforms.smart_crusher import ( - SmartCrusher, - SmartCrusherConfig, - extract_query_anchors, - ) + from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig target_uuid = "550e8400-e29b-41d4-a716-446655440000" @@ -537,10 +526,6 @@ class TestQueryAnchorExtraction: }, ] - # Verify anchor extraction - anchors = extract_query_anchors(f"Get details for request {target_uuid}") - assert target_uuid.lower() in anchors - config = SmartCrusherConfig( enabled=True, min_items_to_analyze=5, @@ -555,7 +540,6 @@ class TestQueryAnchorExtraction: tool_msg = next(m for m in result.messages if m.get("role") == "tool") crushed_content = tool_msg["content"] - # UUID should be preserved assert target_uuid in crushed_content diff --git a/tests/test_ccr.py b/tests/test_ccr.py index d11454a0c..c68880725 100644 --- a/tests/test_ccr.py +++ b/tests/test_ccr.py @@ -235,160 +235,6 @@ class TestCompressionStore: assert store.exists(h) -class TestSmartCrusherCCRIntegration: - """Test SmartCrusher integration with CCR.""" - - @pytest.fixture(autouse=True) - def reset_store(self): - """Reset global store before each test.""" - reset_compression_store() - yield - reset_compression_store() - - def test_compression_caches_original(self): - """SmartCrusher caches original content when compressing.""" - items = [{"id": i, "score": 100 - i, "data": f"item_{i}"} for i in range(100)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=15) - ccr_config = CCRConfig( - enabled=True, - inject_retrieval_marker=False, # Don't add marker for this test - min_items_to_cache=10, - ) - - compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config) - - assert was_modified - - # Check that content was cached - store = get_compression_store() - stats = store.get_stats() - assert stats["entry_count"] >= 1 - - def test_retrieval_marker_injected(self): - """CCR marker is injected when configured.""" - items = [{"id": i, "score": 100 - i, "data": f"item_{i}"} for i in range(100)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=15) - ccr_config = CCRConfig( - enabled=True, - inject_retrieval_marker=True, - min_items_to_cache=10, - ) - - compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config) - - assert was_modified - # Marker should be present - assert "items compressed" in compressed_str or "hash=" in compressed_str - - def test_small_arrays_not_cached(self): - """Arrays smaller than min_items_to_cache are not cached.""" - items = [{"id": i} for i in range(15)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=10) - ccr_config = CCRConfig( - enabled=True, - min_items_to_cache=50, # Require 50+ items - ) - - smart_crush_tool_output(content, config, ccr_config) - - store = get_compression_store() - stats = store.get_stats() - # Should not cache because original has < 50 items - assert stats["entry_count"] == 0 - - def test_uncrushed_data_not_cached(self): - """Data that doesn't get crushed is not cached.""" - # DB results with unique IDs - shouldn't be crushed - items = [{"id": i, "name": f"User {i}", "email": f"user{i}@test.com"} for i in range(30)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=10) - ccr_config = CCRConfig(enabled=True, min_items_to_cache=10) - - compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config) - - # If not modified, shouldn't be cached - if not was_modified: - store = get_compression_store() - stats = store.get_stats() - assert stats["entry_count"] == 0 - - def test_can_retrieve_after_compression(self): - """Can retrieve original content after compression.""" - items = [ - {"id": i, "score": 100 - i, "content": f"Document about topic {i}"} for i in range(100) - ] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=15) - ccr_config = CCRConfig( - enabled=True, - inject_retrieval_marker=True, - min_items_to_cache=10, - ) - - compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config) - - assert was_modified - - # Extract hash from marker - # Marker format: [100 items compressed to 15. Retrieve more: hash=abc123...] - import re - - match = re.search(r"hash=([a-f0-9]+)", compressed_str) - assert match is not None, f"No hash found in: {compressed_str}" - - hash_key = match.group(1) - - # Retrieve original - store = get_compression_store() - entry = store.retrieve(hash_key) - - assert entry is not None - original_items = json.loads(entry.original_content) - assert len(original_items) == 100 - - def test_search_after_compression(self): - """Can search within original content after compression.""" - items = [ - {"id": 1, "content": "Authentication error: invalid token"}, - {"id": 2, "content": "Database connection successful"}, - {"id": 3, "content": "User login completed"}, - ] + [{"id": i, "content": f"Generic log entry {i}"} for i in range(4, 104)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=15) - ccr_config = CCRConfig( - enabled=True, - inject_retrieval_marker=True, - min_items_to_cache=10, - ) - - compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config) - - assert was_modified - - # Extract hash - import re - - match = re.search(r"hash=([a-f0-9]+)", compressed_str) - hash_key = match.group(1) - - # Search for authentication items - store = get_compression_store() - results = store.search(hash_key, "authentication error token") - - # Should find the authentication error item - assert len(results) >= 1 - assert any("Authentication" in r.get("content", "") for r in results) - - class TestCCRConfig: """Test CCR configuration options.""" @@ -402,24 +248,6 @@ class TestCCRConfig: assert config.feedback_enabled is True assert config.min_items_to_cache == 20 - def test_custom_marker_template(self): - """Custom marker template is used.""" - items = [{"id": i, "score": 100 - i} for i in range(100)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=15) - ccr_config = CCRConfig( - enabled=True, - inject_retrieval_marker=True, - min_items_to_cache=10, - marker_template="\n[CUSTOM: {original_count} -> {compressed_count}, key={hash}]", - ) - - compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config) - - if was_modified: - assert "CUSTOM:" in compressed_str or "key=" in compressed_str - class TestCCRFeedbackLoop: """Test CCR feedback tracking for learning.""" diff --git a/tests/test_critical_fixes.py b/tests/test_critical_fixes.py index fd6d8263f..c00a7d0e5 100644 --- a/tests/test_critical_fixes.py +++ b/tests/test_critical_fixes.py @@ -11,8 +11,6 @@ These tests verify the before/after behavior of critical bug fixes: import time from unittest.mock import patch -import pytest - class TestTOINConfidenceMathFix: """Test for CRITICAL: Confidence calculation math error in toin.py:721. @@ -303,146 +301,5 @@ class TestUnboundedStrategyDicts: ) -class TestSmartCrusherTOINIntegration: - """Test for CRITICAL: SmartCrusher not calling toin.record_compression(). - - BUG: SmartCrusher calls feedback.record_compression() but never calls - toin.record_compression(). This means TOIN only learns from retrieval events, - not from compression events - breaking the feedback loop. - - FIX: Add toin.record_compression() call after compression in SmartCrusher. - """ - - def test_smart_crusher_records_to_toin(self): - """SmartCrusher should record compression events to TOIN.""" - from headroom.telemetry.models import ToolSignature - from headroom.telemetry.toin import get_toin, reset_toin - from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig - - reset_toin() - - config = SmartCrusherConfig( - min_items_to_analyze=5, - max_items_after_crush=10, - use_feedback_hints=True, - ) - crusher = SmartCrusher(config) - - # Create test items that look like search results with a clear score field - # This pattern is crushable because: - # 1. Has a clear numeric score field in BOUNDED range [0,1] - # 2. Has repeated structure with some constant fields (type, language) - # 3. Score values vary within the bounded range - items = [ - { - "name": f"repo_{i}", - "relevance_score": (50 - i) / 50.0, # Bounded [0,1] - descending order - "type": "repository", # Constant field - "language": "python" if i % 3 == 0 else "javascript", # Low cardinality - "description": f"Description {i % 5}", # Low cardinality - } - for i in range(50) - ] - - # Get TOIN instance and check initial state - toin = get_toin() - len(toin._patterns) - - # Crush the array - result, info, markers, _summary = crusher._crush_array( - items, query_context="test query", tool_name="test_tool" - ) - - # Verify compression happened (not skipped) - assert "skip" not in info.lower(), ( - f"Compression was skipped: {info}. Test needs crushable data." - ) - - # Get the signature that would have been created - sig = ToolSignature.from_items(items) - - # Check TOIN was notified - with toin._lock: - pattern = toin._patterns.get(sig.structure_hash) - - # After fix, TOIN should have a pattern for this tool's signature - assert pattern is not None, ( - f"TOIN should have recorded the compression event. " - f"Info: {info}, pattern count: {len(toin._patterns)}" - ) - if pattern: - assert pattern.total_compressions >= 1, ( - f"Pattern should have at least 1 compression recorded, got {pattern.total_compressions}" - ) - - class TestAllFixesIntegrated: """Integration tests ensuring all fixes work together.""" - - def test_full_feedback_loop(self): - """Test complete feedback loop: compress -> store -> retrieve -> learn.""" - from headroom.cache.compression_feedback import ( - reset_compression_feedback, - ) - from headroom.cache.compression_store import reset_compression_store - from headroom.telemetry.models import ToolSignature - from headroom.telemetry.toin import get_toin, reset_toin - from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig - - # Reset all singletons - reset_toin() - reset_compression_store() - reset_compression_feedback() - - # Setup - config = SmartCrusherConfig( - min_items_to_analyze=5, - max_items_after_crush=10, - use_feedback_hints=True, - ) - crusher = SmartCrusher(config) - - # Create test items that look like API responses with scoring - # This pattern is crushable because: - # 1. Has a clear numeric score field in BOUNDED range [0,1] - # 2. Has constant fields (status, type) - # 3. Has enough items for compression (100) - items = [ - { - "priority": (100 - i) / 100.0, # Bounded [0,1] - descending order - "status": "ok", # Constant field - "type": "response", # Constant field - "data": f"content_{i % 10}", # Low cardinality (only 10 unique values) - } - for i in range(100) - ] - - # Step 1: Compress - result, info, markers, _summary = crusher._crush_array( - items, query_context="find status", tool_name="api_response" - ) - - # Verify compression happened (not skipped) - assert "skip" not in info.lower(), ( - f"Compression was skipped: {info}. Test needs crushable data." - ) - - # Step 2: Check TOIN was notified (after fix) - toin = get_toin() - sig = ToolSignature.from_items(items) - - with toin._lock: - toin_pattern = toin._patterns.get(sig.structure_hash) - - # After fix, TOIN should have the pattern - assert toin_pattern is not None, ( - f"TOIN should have learned from the compression event. Info: {info}" - ) - assert toin_pattern.total_compressions >= 1, ( - f"TOIN pattern should have recorded compression, got {toin_pattern.total_compressions}" - ) - - -# Run specific test to verify fix -if __name__ == "__main__": - pytest.main([__file__, "-v", "--tb=short"]) diff --git a/tests/test_crushability.py b/tests/test_crushability.py deleted file mode 100644 index e67ac428b..000000000 --- a/tests/test_crushability.py +++ /dev/null @@ -1,423 +0,0 @@ -"""Tests for SmartCrusher crushability analysis. - -These tests verify that SmartCrusher correctly identifies when it's SAFE -to crush data vs when it should SKIP crushing. - -The key insight: High variability + No importance signal = DON'T CRUSH. - -Test scenarios: -1. DB results (unique entities, no signal) → SKIP -2. Search results (has score field) → CRUSH using score -3. Log entries (has errors) → CRUSH keeping errors -4. Time series (has anomalies) → CRUSH keeping anomalies -5. Repetitive data (low uniqueness) → CRUSH with sampling -""" - -import json - -import pytest - -from headroom.transforms.smart_crusher import ( - CompressionStrategy, - SmartAnalyzer, - SmartCrusherConfig, - smart_crush_tool_output, -) - - -class TestCrushabilityDetection: - """Test the crushability analysis logic.""" - - @pytest.fixture - def analyzer(self): - """Create a SmartAnalyzer instance.""" - return SmartAnalyzer(SmartCrusherConfig()) - - def test_db_results_not_crushable(self, analyzer): - """DB query results with unique IDs and no signal should NOT be crushed.""" - # Simulate: SELECT * FROM users LIMIT 50 - items = [ - { - "id": i, - "name": f"User {i}", - "email": f"user{i}@example.com", - "department": "Engineering", - } - for i in range(50) - ] - - analysis = analyzer.analyze_array(items) - - # Should detect unique entities with no importance signal - assert analysis.crushability is not None - assert not analysis.crushability.crushable, ( - f"DB results should NOT be crushable. " - f"Reason: {analysis.crushability.reason}, " - f"Signals: {analysis.crushability.signals_present}" - ) - assert analysis.recommended_strategy == CompressionStrategy.SKIP - assert "unique" in analysis.crushability.reason.lower() - - def test_db_results_with_unique_uuid(self, analyzer): - """DB results with UUID field should NOT be crushed.""" - items = [ - { - "uuid": f"550e8400-e29b-41d4-a716-44665544{i:04d}", - "name": f"Record {i}", - "value": i * 10, - } - for i in range(50) - ] - - analysis = analyzer.analyze_array(items) - - assert analysis.crushability is not None - assert not analysis.crushability.crushable - assert analysis.crushability.has_id_field - - def test_search_results_crushable(self, analyzer): - """Search results with score field SHOULD be crushed.""" - items = [ - { - "id": i, - "title": f"Document {i}", - "snippet": f"This is document {i} content...", - "score": 1.0 - (i * 0.01), # Decreasing relevance - } - for i in range(100) - ] - - analysis = analyzer.analyze_array(items) - - # Should detect score field as importance signal - assert analysis.crushability is not None - assert analysis.crushability.crushable, ( - f"Search results should be crushable. Reason: {analysis.crushability.reason}" - ) - assert analysis.crushability.has_score_field - assert any("score" in s for s in analysis.crushability.signals_present) - - def test_log_entries_with_errors_crushable(self, analyzer): - """Log entries containing structural outliers SHOULD be crushed (outliers preserved).""" - items = [] - for i in range(100): - item = { - "id": i, - "timestamp": f"2024-01-15T10:{i:02d}:00Z", - "message": f"Request processed successfully - {i}", - "level": "INFO", - } - # Add some errors - these are STRUCTURAL OUTLIERS (have extra "error" field) - if i % 20 == 0: - item["level"] = "ERROR" - item["message"] = f"Connection failed: timeout at {i}" - item["error"] = "TimeoutError" # Extra field that most items don't have - items.append(item) - - analysis = analyzer.analyze_array(items) - - # Should detect structural outliers (items with rare fields like "error") - assert analysis.crushability is not None - assert analysis.crushability.crushable - # Now uses structural_outliers instead of keyword-based error count - assert any( - "structural_outliers" in s or "outlier" in s.lower() - for s in analysis.crushability.signals_present - ) - - def test_time_series_with_anomalies_crushable(self, analyzer): - """Time series with numeric anomalies SHOULD be crushed.""" - items = [] - for i in range(100): - value = 100.0 # Normal value - if i in [25, 50, 75]: # Anomaly points - value = 999.0 - items.append( - { - "id": i, - "timestamp": i, - "cpu_usage": value, - } - ) - - analysis = analyzer.analyze_array(items) - - # Should detect anomalies as importance signal - assert analysis.crushability is not None - assert analysis.crushability.crushable - assert analysis.crushability.anomaly_count > 0 - - def test_repetitive_data_crushable(self, analyzer): - """Repetitive data (low uniqueness) SHOULD be crushable.""" - # Same status repeated many times - items = [ - { - "id": i, - "status": "success", # Same for all - "code": 200, # Same for all - "message": "OK", # Same for all - } - for i in range(100) - ] - - analysis = analyzer.analyze_array(items) - - # Should detect low uniqueness - safe to sample - assert analysis.crushability is not None - assert analysis.crushability.crushable - # Can be "low_uniqueness" or "repetitive_content_with_ids" - assert ( - "low_uniqueness" in analysis.crushability.reason - or "repetitive" in analysis.crushability.reason - ) - - def test_file_listing_not_crushable(self, analyzer): - """File listing with unique paths should NOT be crushed.""" - items = [ - { - "id": i, - "path": f"/home/user/project/src/module{i}/file{i}.py", - "size": 1000 + i, - "modified": f"2024-01-{(i % 28) + 1:02d}", - } - for i in range(50) - ] - - analysis = analyzer.analyze_array(items) - - # Paths are highly unique, no importance signal - assert analysis.crushability is not None - # Should NOT crush file listings - assert not analysis.crushability.crushable or analysis.crushability.confidence < 0.7 - - def test_order_list_not_crushable(self, analyzer): - """Order list with unique order IDs should NOT be crushed.""" - items = [ - { - "order_id": f"ORD-2024-{i:05d}", - "customer": f"Customer {i}", - "total": 50.0 + i, - "status": "completed", - } - for i in range(50) - ] - - analysis = analyzer.analyze_array(items) - - # Each order is a unique entity - assert analysis.crushability is not None - # order_id contains 'id' pattern - assert not analysis.crushability.crushable - - -class TestCrushabilityEndToEnd: - """End-to-end tests for crushability-aware crushing.""" - - def test_db_results_preserved_completely(self): - """DB results should be returned unchanged when not crushable.""" - items = [{"id": i, "name": f"User {i}", "email": f"user{i}@test.com"} for i in range(30)] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=10) - crushed, was_modified, info = smart_crush_tool_output(content, config) - - # Should NOT be modified (skip crushing) - if was_modified: - result = json.loads(crushed) - # If it was modified, all items should still be there - assert len(result) == 30, ( - f"DB results should not lose items! Had 30, got {len(result)}. Info: {info}" - ) - - def test_search_results_crushed_by_score(self): - """Search results should be crushed using score field.""" - items = [ - { - "id": i, - "title": f"Result {i}", - "score": 100 - i, # Higher score = more relevant - } - for i in range(100) - ] - content = json.dumps(items) - - config = SmartCrusherConfig(max_items_after_crush=15) - crushed, was_modified, info = smart_crush_tool_output(content, config) - - assert was_modified - result = json.loads(crushed) - assert len(result) < 100 - - # Top scores should be preserved - scores = [item.get("score", 0) for item in result] - assert max(scores) >= 90 # Top items preserved - - def test_mixed_data_with_errors_preserves_errors(self): - """Data with errors should crush but preserve ALL errors.""" - items = [] - error_ids = [5, 25, 45, 65, 85] - for i in range(100): - item = {"id": i, "data": f"value_{i}"} - if i in error_ids: - item["status"] = "failed" - item["error"] = f"Error at {i}" - items.append(item) - - content = json.dumps(items) - config = SmartCrusherConfig(max_items_after_crush=20) - crushed, was_modified, info = smart_crush_tool_output(content, config) - - result = json.loads(crushed) - - # All errors must be preserved - error_count = sum(1 for item in result if item.get("error")) - assert error_count == len(error_ids), ( - f"All {len(error_ids)} errors should be preserved, got {error_count}" - ) - - -class TestCrushabilitySignals: - """Test individual signal detection.""" - - @pytest.fixture - def analyzer(self): - return SmartAnalyzer(SmartCrusherConfig()) - - def test_detects_id_field_variations(self, analyzer): - """Should detect various ID field naming patterns.""" - test_cases = [ - ("id", [{"id": i} for i in range(20)]), - ("uuid", [{"uuid": f"uuid-{i}"} for i in range(20)]), - ("_id", [{"_id": f"mongo-{i}"} for i in range(20)]), - ("pk", [{"pk": i} for i in range(20)]), - ("key", [{"key": f"key-{i}"} for i in range(20)]), - ("user_id", [{"user_id": i} for i in range(20)]), - ] - - for field_name, items in test_cases: - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - assert analysis.crushability.has_id_field, f"Should detect '{field_name}' as ID field" - - def test_detects_score_field_variations(self, analyzer): - """Should detect various score field naming patterns.""" - test_cases = [ - "score", - "rank", - "relevance", - "confidence", - "_score", - "rating", - ] - - for field_name in test_cases: - items = [{field_name: i * 0.1, "data": f"item_{i}"} for i in range(20)] - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - assert analysis.crushability.has_score_field, ( - f"Should detect '{field_name}' as score field" - ) - - def test_detects_error_keywords(self, analyzer): - """Should detect various error keyword patterns.""" - error_keywords = ["error", "exception", "failed", "failure", "critical", "fatal"] - - for keyword in error_keywords: - items = [{"id": i, "msg": "OK"} for i in range(20)] - items[10]["msg"] = f"Something {keyword} happened" - - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - assert analysis.crushability.error_item_count >= 1, ( - f"Should detect '{keyword}' as error indicator" - ) - - -class TestCrushabilityEdgeCases: - """Test edge cases in crushability analysis.""" - - @pytest.fixture - def analyzer(self): - return SmartAnalyzer(SmartCrusherConfig()) - - def test_empty_array(self, analyzer): - """Empty array should not crash.""" - analysis = analyzer.analyze_array([]) - assert analysis.recommended_strategy == CompressionStrategy.NONE - - def test_small_array_skipped(self, analyzer): - """Arrays below min_items_to_analyze should be skipped.""" - items = [{"id": i} for i in range(3)] - analysis = analyzer.analyze_array(items) - assert analysis.recommended_strategy == CompressionStrategy.NONE - - def test_mixed_signals(self, analyzer): - """Data with multiple signals should still be crushable.""" - items = [] - for i in range(100): - item = { - "id": i, - "score": 100 - i, # Score signal - "value": 50.0, - } - if i == 50: - item["error"] = "Test error" # Error signal - item["value"] = 999.0 # Anomaly signal - items.append(item) - - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - assert analysis.crushability.crushable - assert len(analysis.crushability.signals_present) >= 2 - - def test_all_items_are_errors(self, analyzer): - """When all items are errors, keyword detection finds them as a signal. - - With keyword-based error detection (for the preservation guarantee), - when ALL items have error keywords, we detect error_keywords:50 as a - signal. This makes the data technically crushable. - - However, since ALL items are errors, they will ALL be preserved due to - the preservation guarantee. The end result is the same - no data loss. - """ - items = [{"id": i, "error": f"Error {i}", "status": "failed"} for i in range(50)] - - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - - # With keyword-based error detection, all 50 items contain error keywords - # This IS a signal (error_keywords:50), making the data crushable. - # However, all 50 items will be preserved due to the preservation guarantee. - assert analysis.crushability.crushable - assert "error_keywords:50" in analysis.crushability.signals_present - - -class TestCrushabilityConfidence: - """Test confidence scoring in crushability analysis.""" - - @pytest.fixture - def analyzer(self): - return SmartAnalyzer(SmartCrusherConfig()) - - def test_high_confidence_for_clear_cases(self, analyzer): - """Clear-cut cases should have high confidence.""" - # Low uniqueness - clearly safe - items = [{"status": "ok", "code": 200} for _ in range(100)] - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - assert analysis.crushability.confidence >= 0.8 - - def test_lower_confidence_for_ambiguous_cases(self, analyzer): - """Ambiguous cases should have lower confidence.""" - # Medium uniqueness with weak signal - items = [ - {"id": i, "value": i % 10, "status": "active" if i % 2 == 0 else "inactive"} - for i in range(100) - ] - # Add one error to provide weak signal - items[50]["error"] = "minor issue" - - analysis = analyzer.analyze_array(items) - assert analysis.crushability is not None - # Should be lower confidence due to ambiguity - assert analysis.crushability.confidence <= 0.7 diff --git a/tests/test_relevance.py b/tests/test_relevance.py index bdfa4de1a..6da26e1a0 100644 --- a/tests/test_relevance.py +++ b/tests/test_relevance.py @@ -273,33 +273,6 @@ class TestEmbeddingAvailable: class TestSmartCrusherIntegration: """Integration tests for SmartCrusher with RelevanceScorer.""" - def test_crusher_uses_scorer(self): - """SmartCrusher uses relevance scorer for context matching.""" - from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig - - config = SmartCrusherConfig( - min_items_to_analyze=3, - min_tokens_to_crush=0, - max_items_after_crush=5, - ) - crusher = SmartCrusher(config=config) - - # Verify scorer is initialized - assert hasattr(crusher, "_scorer") - assert hasattr(crusher, "_relevance_threshold") - - def test_custom_scorer_injection(self): - """Custom scorer can be injected.""" - from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig - - custom_scorer = BM25Scorer(k1=2.0) - crusher = SmartCrusher( - config=SmartCrusherConfig(), - scorer=custom_scorer, - ) - - assert crusher._scorer is custom_scorer - def test_context_extraction(self): """Context is extracted from messages.""" from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig diff --git a/tests/test_relevance_extra.py b/tests/test_relevance_extra.py index 62a8fa895..90b6d34e0 100644 --- a/tests/test_relevance_extra.py +++ b/tests/test_relevance_extra.py @@ -90,7 +90,7 @@ def test_embedding_numpy_and_model_error_paths(monkeypatch) -> None: assert embedding._cosine_similarity([1, 0], [0, 1]) == 0.0 monkeypatch.setattr(EmbeddingScorer, "is_available", classmethod(lambda cls: False)) - with pytest.raises(RuntimeError, match="requires sentence-transformers"): + with pytest.raises(RuntimeError, match="requires fastembed"): EmbeddingScorer()._get_model() diff --git a/tests/test_toin_field_learning.py b/tests/test_toin_field_learning.py deleted file mode 100644 index ae875cd43..000000000 --- a/tests/test_toin_field_learning.py +++ /dev/null @@ -1,737 +0,0 @@ -"""Tests for TOIN Field-Level Learning. - -These tests PROVE that field-level learning actually works: -1. FieldSemantics correctly infers types from retrieval patterns -2. TOIN populates field_semantics from retrievals -3. SmartCrusher uses learned semantics to detect important items -4. End-to-end: important items are preserved based on learned behavior - -No hardcoded patterns - all learning is behavior-based. -""" - -import hashlib - -import pytest - -from headroom.telemetry import ( - ToolIntelligenceNetwork, - ToolPattern, - ToolSignature, - reset_toin, -) -from headroom.telemetry.models import FieldSemantics - - -@pytest.fixture(autouse=True) -def reset_globals(): - """Reset global state before each test.""" - reset_toin() - yield - reset_toin() - - -def _hash_value(value) -> str: - """Hash a value the same way TOIN does.""" - value_str = str(value) - return hashlib.sha256(value_str.encode()).hexdigest()[:8] - - -def _hash_field(field_name: str) -> str: - """Hash a field name the same way TOIN does.""" - return hashlib.sha256(field_name.encode()).hexdigest()[:8] - - -class TestFieldSemanticsLearning: - """Test that FieldSemantics correctly learns from retrieval patterns.""" - - def test_identifier_type_inference(self): - """PROVES: Field used with exact-match queries + high uniqueness = identifier.""" - fs = FieldSemantics(field_hash="test123") - - # Simulate: user retrieves items by unique IDs (exact match queries) - # Each ID is different - high uniqueness - for i in range(10): - fs.record_retrieval_value(_hash_value(f"id_{i}"), operator="=") - - # Simulate compression stats: all values are unique - fs.record_compression_stats( - unique_values=100, - total_values=100, # uniqueness ratio = 1.0 - most_common_value_hash=_hash_value("id_0"), - most_common_frequency=0.01, # No dominant value - ) - fs.record_compression_stats( - unique_values=100, - total_values=100, - most_common_value_hash=_hash_value("id_1"), - most_common_frequency=0.01, - ) - - # Now infer the type - fs.infer_type() - - # VERIFY: Should be classified as identifier - assert fs.inferred_type == "identifier", ( - f"Expected 'identifier' but got '{fs.inferred_type}'. " - "High uniqueness + exact match queries should = identifier" - ) - assert fs.confidence > 0.5, "Should have reasonable confidence" - - def test_error_indicator_type_inference(self): - """PROVES: Field with dominant default + retrievals for non-default = error_indicator.""" - fs = FieldSemantics(field_hash="status_field") - - # Simulate: most items have status="success" (the default) - # But user only retrieves items with status="error" or "failed" - error_hash = _hash_value("error") - failed_hash = _hash_value("failed") - success_hash = _hash_value("success") - - # User retrieves "error" and "failed" values (non-default) - for _ in range(5): - fs.record_retrieval_value(error_hash, operator="=") - fs.record_retrieval_value(failed_hash, operator="=") - - # Compression stats: 90% have "success" (the default) - fs.record_compression_stats( - unique_values=3, # "success", "error", "failed" - total_values=100, - most_common_value_hash=success_hash, - most_common_frequency=0.9, # 90% are "success" - ) - fs.record_compression_stats( - unique_values=3, - total_values=100, - most_common_value_hash=success_hash, - most_common_frequency=0.9, - ) - - fs.infer_type() - - # VERIFY: Should be error_indicator - assert fs.inferred_type == "error_indicator", ( - f"Expected 'error_indicator' but got '{fs.inferred_type}'. " - "Dominant default + retrieval of non-default = error indicator" - ) - assert fs.default_value_hash == success_hash, "Default should be 'success'" - assert fs.confidence > 0.5, "Should have reasonable confidence" - - def test_status_type_inference(self): - """PROVES: Low cardinality + specific values retrieved = status.""" - fs = FieldSemantics(field_hash="state_field") - - # Simulate: field has few unique values (low cardinality) - # User retrieves the same few values repeatedly - pending_hash = _hash_value("pending") - active_hash = _hash_value("active") - - for _ in range(6): - fs.record_retrieval_value(pending_hash, operator="=") - for _ in range(4): - fs.record_retrieval_value(active_hash, operator="=") - - # Compression stats: only 5 unique values across 100 items - fs.record_compression_stats( - unique_values=5, - total_values=100, # uniqueness ratio = 0.05 (very low) - most_common_value_hash=None, - most_common_frequency=0.3, # No overwhelming default - ) - fs.record_compression_stats( - unique_values=5, - total_values=100, - most_common_value_hash=None, - most_common_frequency=0.3, - ) - - fs.infer_type() - - # VERIFY: Should be status - assert fs.inferred_type == "status", ( - f"Expected 'status' but got '{fs.inferred_type}'. " - "Low cardinality + specific values retrieved = status" - ) - - def test_score_type_inference(self): - """PROVES: Range queries = score type.""" - fs = FieldSemantics(field_hash="relevance_field") - - # Simulate: user queries with range operators (top-N behavior) - for _ in range(8): - fs.record_retrieval_value(_hash_value("0.95"), operator=">") - for _ in range(4): - fs.record_retrieval_value(_hash_value("0.90"), operator=">=") - - # Compression stats - fs.record_compression_stats( - unique_values=50, - total_values=100, - most_common_value_hash=None, - most_common_frequency=0.1, - ) - fs.record_compression_stats( - unique_values=50, - total_values=100, - most_common_value_hash=None, - most_common_frequency=0.1, - ) - - fs.infer_type() - - # VERIFY: Should be score - assert fs.inferred_type == "score", ( - f"Expected 'score' but got '{fs.inferred_type}'. " - "Range queries (>, >=) should = score type" - ) - - def test_content_type_inference(self): - """PROVES: Contains/text search queries = content type.""" - fs = FieldSemantics(field_hash="description_field") - - # Simulate: user does text search on this field - for i in range(10): - fs.record_retrieval_value(_hash_value(f"search_term_{i}"), operator="contains") - - # Compression stats: high uniqueness (different descriptions) - fs.record_compression_stats( - unique_values=90, - total_values=100, - most_common_value_hash=None, - most_common_frequency=0.05, - ) - fs.record_compression_stats( - unique_values=90, - total_values=100, - most_common_value_hash=None, - most_common_frequency=0.05, - ) - - fs.infer_type() - - # VERIFY: Should be content - assert fs.inferred_type == "content", ( - f"Expected 'content' but got '{fs.inferred_type}'. " - "Contains queries should = content type" - ) - - def test_is_value_important_for_error_indicator(self): - """PROVES: For error_indicator, non-default values are important.""" - fs = FieldSemantics(field_hash="status") - - error_hash = _hash_value("error") - success_hash = _hash_value("success") - - # Set up as error_indicator - fs.inferred_type = "error_indicator" - fs.confidence = 0.8 - fs.default_value_hash = success_hash - fs.important_value_hashes = [error_hash] - - # VERIFY - assert fs.is_value_important(error_hash), "Error value should be important" - assert not fs.is_value_important(success_hash), "Default value should NOT be important" - - def test_is_value_important_for_status(self): - """PROVES: For status fields, retrieved values are important.""" - fs = FieldSemantics(field_hash="state") - - pending_hash = _hash_value("pending") - unknown_hash = _hash_value("never_retrieved") - - # Set up as status - fs.inferred_type = "status" - fs.confidence = 0.7 - fs.value_retrieval_frequency = {pending_hash: 5} - - # VERIFY - assert fs.is_value_important(pending_hash), "Retrieved value should be important" - assert not fs.is_value_important(unknown_hash), ( - "Never-retrieved value should NOT be important" - ) - - -class TestTOINFieldLearningIntegration: - """Test that TOIN correctly integrates field-level learning.""" - - def test_record_retrieval_populates_field_semantics(self): - """PROVES: record_retrieval with items actually populates field_semantics.""" - toin = ToolIntelligenceNetwork() - - # Create a tool signature - items = [ - {"id": "123", "status": "ok", "value": 100}, - {"id": "456", "status": "error", "value": 200}, - ] - sig = ToolSignature.from_items(items) - - # Record retrieval with items - THIS IS WHERE LEARNING HAPPENS - toin.record_retrieval( - tool_signature_hash=sig.structure_hash, - retrieval_type="full", - query="status=error", - query_fields=["status"], - retrieved_items=items, - ) - - # VERIFY: pattern should have field_semantics populated - pattern = toin._patterns.get(sig.structure_hash) - assert pattern is not None, "Pattern should exist" - assert len(pattern.field_semantics) > 0, ( - f"field_semantics should be populated after retrieval. Got: {pattern.field_semantics}" - ) - - # Check that field hashes match expected fields - expected_field_hashes = {_hash_field(f) for f in ["id", "status", "value"]} - actual_field_hashes = set(pattern.field_semantics.keys()) - assert expected_field_hashes == actual_field_hashes, ( - f"Expected field hashes {expected_field_hashes}, got {actual_field_hashes}" - ) - - def test_repeated_retrievals_trigger_type_inference(self): - """PROVES: After multiple retrievals, TOIN infers field types.""" - toin = ToolIntelligenceNetwork() - - # Simulate items with status field - items = [ - {"status": "success"}, - {"status": "success"}, - {"status": "success"}, - {"status": "error"}, - ] - sig = ToolSignature.from_items(items) - - # Simulate 6 retrievals (enough to trigger inference at retrieval 5) - for _ in range(6): - # User always retrieves items with error status - toin.record_retrieval( - tool_signature_hash=sig.structure_hash, - retrieval_type="full", - query="status=error", - query_fields=["status"], - retrieved_items=[{"status": "error"}], - ) - # Also record compression to get enough data for inference - toin.record_compression( - tool_signature=sig, - original_count=100, - compressed_count=10, - original_tokens=1000, - compressed_tokens=100, - strategy="top_n", - items=items, - ) - - # VERIFY: After enough retrievals, type should be inferred - pattern = toin._patterns.get(sig.structure_hash) - status_hash = _hash_field("status") - assert status_hash in pattern.field_semantics, "Status field should be tracked" - - status_sem = pattern.field_semantics[status_hash] - # The type should be inferred (not unknown) after enough data - assert status_sem.retrieval_count >= 6, ( - f"Should have 6+ retrievals, got {status_sem.retrieval_count}" - ) - - def test_get_recommendation_includes_field_semantics(self): - """PROVES: get_recommendation returns learned field_semantics.""" - toin = ToolIntelligenceNetwork() - - # Set up pattern with learned field_semantics - items = [{"id": "123", "status": "ok"}] - sig = ToolSignature.from_items(items) - - # Record enough data - for i in range(10): - toin.record_retrieval( - tool_signature_hash=sig.structure_hash, - retrieval_type="full", - query=f"id={i}", - query_fields=["id"], - retrieved_items=[{"id": str(i), "status": "ok"}], - ) - toin.record_compression( - tool_signature=sig, - original_count=100, - compressed_count=10, - original_tokens=1000, - compressed_tokens=100, - strategy="top_n", - items=items, - ) - - # Get recommendation - hint = toin.get_recommendation(sig) - - # VERIFY: hint should include field_semantics - assert hint is not None, "Should get a recommendation" - # field_semantics might be empty if confidence is too low, - # but the attribute should exist - assert hasattr(hint, "field_semantics"), "Hint should have field_semantics attribute" - - def test_field_semantics_persisted_correctly(self): - """PROVES: field_semantics survives to_dict/from_dict round-trip.""" - # ToolPattern is already imported at module level from headroom.telemetry - - # Create pattern with field_semantics - pattern = ToolPattern(tool_signature_hash="test123") - - # Add field semantics - fs = FieldSemantics(field_hash="field123") - fs.inferred_type = "error_indicator" - fs.confidence = 0.8 - fs.important_value_hashes = ["value1", "value2"] - fs.default_value_hash = "default" - pattern.field_semantics["field123"] = fs - - # Round-trip through dict - d = pattern.to_dict() - pattern2 = ToolPattern.from_dict(d) - - # VERIFY: field_semantics preserved - assert "field123" in pattern2.field_semantics, "field_semantics should be preserved" - fs2 = pattern2.field_semantics["field123"] - assert fs2.inferred_type == "error_indicator" - assert fs2.confidence == 0.8 - assert fs2.important_value_hashes == ["value1", "value2"] - - -class TestSmartCrusherUsesLearnedSemantics: - """Test that SmartCrusher actually uses learned field semantics.""" - - def test_detect_items_by_learned_semantics_finds_important_items(self): - """PROVES: _detect_items_by_learned_semantics correctly identifies items.""" - from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics - - # Create field semantics that knows "error" is important - status_hash = _hash_field("status") - error_hash = _hash_value("error") - success_hash = _hash_value("success") - - fs = FieldSemantics(field_hash=status_hash) - fs.inferred_type = "error_indicator" - fs.confidence = 0.8 - fs.default_value_hash = success_hash - fs.important_value_hashes = [error_hash] - fs.value_retrieval_frequency = {error_hash: 10} - - field_semantics = {status_hash: fs} - - # Test items - index 1 has error status - items = [ - {"status": "success", "message": "all good"}, - {"status": "error", "message": "something failed"}, # <-- This should be detected - {"status": "success", "message": "also good"}, - ] - - # VERIFY - important_indices = _detect_items_by_learned_semantics(items, field_semantics) - assert 1 in important_indices, ( - f"Index 1 (error status) should be detected as important. " - f"Got indices: {important_indices}" - ) - assert 0 not in important_indices, "Index 0 (success) should not be important" - assert 2 not in important_indices, "Index 2 (success) should not be important" - - def test_detect_items_handles_empty_semantics(self): - """PROVES: Function handles edge cases gracefully.""" - from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics - - items = [{"status": "ok"}] - - # Empty semantics - assert _detect_items_by_learned_semantics(items, {}) == [] - assert _detect_items_by_learned_semantics(items, None) == [] - assert _detect_items_by_learned_semantics([], {"x": FieldSemantics(field_hash="x")}) == [] - - def test_detect_items_requires_confidence(self): - """PROVES: Low confidence semantics are ignored.""" - from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics - - status_hash = _hash_field("status") - error_hash = _hash_value("error") - - fs = FieldSemantics(field_hash=status_hash) - fs.inferred_type = "error_indicator" - fs.confidence = 0.1 # TOO LOW - fs.important_value_hashes = [error_hash] - - items = [{"status": "error"}] - - # VERIFY: Low confidence = ignored - result = _detect_items_by_learned_semantics(items, {status_hash: fs}) - assert result == [], "Low confidence semantics should be ignored" - - -class TestEndToEndFieldLearning: - """End-to-end tests proving the full learning pipeline works.""" - - def test_full_pipeline_learns_and_applies(self): - """PROVES: End-to-end learning from retrieval to compression.""" - toin = ToolIntelligenceNetwork() - - # PHASE 1: Learning - User retrieves items with error status - # Simulating: "show me all failed items" - items_with_errors = [ - {"id": "1", "status": "success", "data": "..."}, - {"id": "2", "status": "error", "data": "..."}, # Retrieved - {"id": "3", "status": "success", "data": "..."}, - {"id": "4", "status": "failed", "data": "..."}, # Retrieved - ] - sig = ToolSignature.from_items(items_with_errors) - - # User keeps retrieving error/failed items (learning behavior) - for _ in range(5): - toin.record_retrieval( - tool_signature_hash=sig.structure_hash, - retrieval_type="full", - query="status!=success", - query_fields=["status"], - retrieved_items=[ - {"id": "2", "status": "error", "data": "..."}, - {"id": "4", "status": "failed", "data": "..."}, - ], - ) - toin.record_compression( - tool_signature=sig, - original_count=100, - compressed_count=10, - original_tokens=1000, - compressed_tokens=100, - strategy="top_n", - items=[ - {"id": str(i), "status": "success" if i % 5 != 0 else "error", "data": "..."} - for i in range(100) - ], - ) - - # PHASE 2: Verify learning occurred - pattern = toin._patterns.get(sig.structure_hash) - assert pattern is not None - assert len(pattern.field_semantics) > 0, "Should have learned field semantics" - - # Check status field was learned - status_hash = _hash_field("status") - if status_hash in pattern.field_semantics: - status_sem = pattern.field_semantics[status_hash] - # Error value should be tracked - error_hash = _hash_value("error") - failed_hash = _hash_value("failed") - assert ( - error_hash in status_sem.important_value_hashes - or failed_hash in status_sem.important_value_hashes - ), "Error/failed values should be marked as important" - - def test_recommendation_hint_includes_learned_semantics(self): - """PROVES: TOIN recommendation includes learned field semantics for SmartCrusher.""" - toin = ToolIntelligenceNetwork() - - # Set up sufficient learning - items = [{"status": "ok", "id": "123"}] - sig = ToolSignature.from_items(items) - - # Create pattern with confident field semantics by directly manipulating internal state - # (This is a test - in real code, patterns are created via record_* methods) - pattern = ToolPattern(tool_signature_hash=sig.structure_hash) - toin._patterns[sig.structure_hash] = pattern - - status_hash = _hash_field("status") - fs = FieldSemantics(field_hash=status_hash) - fs.inferred_type = "error_indicator" - fs.confidence = 0.8 # High confidence - fs.retrieval_count = 10 - fs.important_value_hashes = [_hash_value("error")] - pattern.field_semantics[status_hash] = fs - - # Ensure pattern has enough data for recommendation - pattern.total_compressions = 10 - pattern.sample_size = 10 # Required by min_samples_for_recommendation - pattern.confidence = 0.5 - - # Get recommendation - hint = toin.get_recommendation(sig) - - # VERIFY - assert hint is not None - assert len(hint.field_semantics) > 0, ( - f"Recommendation should include field_semantics. Got: {hint.field_semantics}" - ) - assert status_hash in hint.field_semantics - - -class TestFieldSemanticsMemoryBounds: - """Test that memory bounds are enforced.""" - - def test_important_values_bounded(self): - """PROVES: important_value_hashes stays within bounds.""" - fs = FieldSemantics(field_hash="test") - - # Add more values than MAX_IMPORTANT_VALUES - for i in range(fs.MAX_IMPORTANT_VALUES + 20): - fs.record_retrieval_value(_hash_value(f"value_{i}")) - - # VERIFY: bounded - assert len(fs.important_value_hashes) <= fs.MAX_IMPORTANT_VALUES - - def test_value_frequency_bounded(self): - """PROVES: value_retrieval_frequency stays within bounds.""" - fs = FieldSemantics(field_hash="test") - - # Add more values than MAX_VALUE_FREQUENCY_ENTRIES - for i in range(fs.MAX_VALUE_FREQUENCY_ENTRIES + 20): - fs.record_retrieval_value(_hash_value(f"value_{i}")) - - # VERIFY: bounded - assert len(fs.value_retrieval_frequency) <= fs.MAX_VALUE_FREQUENCY_ENTRIES - - -class TestProductionCodePath: - """Integration tests for the ACTUAL production code path. - - These tests verify that CompressionStore -> TOIN integration works, - not just TOIN in isolation. This is critical because the unit tests - can pass while the production integration is broken. - """ - - def test_compression_store_passes_items_to_toin(self): - """PROVES: CompressionStore.process_pending_feedback passes retrieved_items to TOIN. - - This is the integration test that would have caught the original bug - where retrieved_items was never passed to TOIN in production. - """ - import json - - from headroom.cache.compression_store import CompressionStore - from headroom.telemetry.toin import get_toin, reset_toin - - reset_toin() - toin = get_toin() - - # Create a store with feedback enabled - store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True) - - # Store some compressed content with items that have distinct field values - items = [ - {"id": "123", "status": "success", "value": 100}, - {"id": "456", "status": "error", "value": 200}, - {"id": "789", "status": "success", "value": 300}, - ] - compressed_json = json.dumps(items) - original_json = json.dumps(items * 10) # Original was bigger - - # Create a tool signature hash for this structure - sig = ToolSignature.from_items(items) - - hash_key = store.store( - original=original_json, - compressed=compressed_json, - original_tokens=1000, - compressed_tokens=100, - original_item_count=30, - compressed_item_count=3, - tool_name="test_api", - tool_call_id="call_123", - tool_signature_hash=sig.structure_hash, - compression_strategy="top_n", - ) - - # Simulate a retrieval (this triggers the feedback loop) - store.retrieve(hash_key, query="status=error") - - # Process pending feedback - THIS IS WHERE THE BUG WAS - store.process_pending_feedback() - - # VERIFY: TOIN should have received the items and learned from them - pattern = toin._patterns.get(sig.structure_hash) - assert pattern is not None, "TOIN should have a pattern for this tool" - - # The key assertion: field_semantics should be populated - # This would have FAILED before the fix because retrieved_items wasn't passed - assert len(pattern.field_semantics) > 0, ( - "TOIN should have learned field semantics from the retrieved items. " - "If this fails, CompressionStore is not passing retrieved_items to TOIN." - ) - - # Verify specific fields were learned - id_hash = _hash_field("id") - status_hash = _hash_field("status") - value_hash = _hash_field("value") - - learned_fields = set(pattern.field_semantics.keys()) - expected_fields = {id_hash, status_hash, value_hash} - assert expected_fields == learned_fields, ( - f"Expected fields {expected_fields}, got {learned_fields}" - ) - - def test_compression_store_handles_wrapped_arrays(self): - """PROVES: CompressionStore correctly extracts items from wrapped arrays.""" - import json - - from headroom.cache.compression_store import CompressionStore - from headroom.telemetry.toin import get_toin, reset_toin - - reset_toin() - toin = get_toin() - - store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True) - - # Content wrapped in {"results": [...]} pattern - items = [{"name": "test", "score": 0.95}] - wrapped_content = json.dumps({"results": items, "total": 1}) - - sig = ToolSignature.from_items(items) - - hash_key = store.store( - original=wrapped_content, - compressed=wrapped_content, - original_tokens=100, - compressed_tokens=100, - original_item_count=1, - compressed_item_count=1, - tool_name="search_api", - tool_call_id="call_456", - tool_signature_hash=sig.structure_hash, - compression_strategy="top_n", - ) - - store.retrieve(hash_key) - store.process_pending_feedback() - - # VERIFY: Items were extracted from wrapped structure - pattern = toin._patterns.get(sig.structure_hash) - assert pattern is not None - assert len(pattern.field_semantics) > 0, ( - "TOIN should extract items from wrapped arrays like {'results': [...]}" - ) - - def test_compression_store_handles_invalid_json(self): - """PROVES: CompressionStore gracefully handles invalid JSON.""" - from headroom.cache.compression_store import CompressionStore - from headroom.telemetry.toin import get_toin, reset_toin - - reset_toin() - get_toin() # Initialize TOIN for feedback loop - - store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True) - - # Store invalid JSON content - invalid_json = "not valid json {" - - hash_key = store.store( - original=invalid_json, - compressed=invalid_json, - original_tokens=10, - compressed_tokens=10, - original_item_count=0, - compressed_item_count=0, - tool_name="broken_api", - tool_call_id="call_789", - tool_signature_hash="invalid123", - compression_strategy="none", - ) - - # This should not crash - store.retrieve(hash_key) - store.process_pending_feedback() - - # VERIFY: No crash, pattern may or may not exist but no exception - # The main assertion is that we got here without exception diff --git a/tests/test_toin_integration.py b/tests/test_toin_integration.py index 4770aa87f..860228781 100644 --- a/tests/test_toin_integration.py +++ b/tests/test_toin_integration.py @@ -8,7 +8,6 @@ Tests the complete flow: 5. Future compressions get improved recommendations """ -import json import tempfile from pathlib import Path @@ -18,18 +17,11 @@ from headroom.cache.compression_store import ( get_compression_store, reset_compression_store, ) -from headroom.config import CCRConfig -from headroom.telemetry import ToolSignature from headroom.telemetry.toin import ( TOINConfig, - ToolIntelligenceNetwork, get_toin, reset_toin, ) -from headroom.transforms.smart_crusher import ( - SmartCrusher, - SmartCrusherConfig, -) @pytest.fixture @@ -55,332 +47,3 @@ def fresh_store(): store = get_compression_store(max_entries=100, default_ttl=300) yield store reset_compression_store() - - -class TestTOINIntegration: - """Test the full TOIN feedback loop.""" - - def test_compression_records_correct_hash(self, fresh_toin, fresh_store): - """Test that SmartCrusher records the correct tool_signature_hash in store.""" - # Create test data - items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)] - content = json.dumps(items) - - # Create a SmartCrusher with CCR enabled - ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False) - crusher = SmartCrusher( - SmartCrusherConfig(max_items_after_crush=10), - ccr_config=ccr_config, - ) - - # Compress the content - crushed, was_modified, info = crusher._smart_crush_content(content, tool_name="test_tool") - - assert was_modified, "Content should be modified by compression" - - # Verify the store has an entry with the correct tool_signature_hash - stats = fresh_store.get_stats() - assert stats["entry_count"] >= 1, "Store should have at least one entry" - - # Get the entry and verify it has tool_signature_hash - # We need to find the hash key from the store - entries = [entry for _, entry in fresh_store._backend.items()] - assert len(entries) >= 1, "Should have at least one entry" - - entry = entries[0] - assert entry.tool_signature_hash is not None, "Entry should have tool_signature_hash" - assert entry.compression_strategy is not None, "Entry should have compression_strategy" - - # Verify the hash matches what ToolSignature would generate - expected_signature = ToolSignature.from_items(items) - assert entry.tool_signature_hash == expected_signature.structure_hash, ( - "Stored hash should match ToolSignature.structure_hash" - ) - - def test_retrieval_updates_toin_strategy_success(self, fresh_toin, fresh_store): - """Test that retrieval events update TOIN strategy success rates.""" - # Create test data - items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)] - signature = ToolSignature.from_items(items) - - # Record some compressions with fresh_toin - for _ in range(5): - fresh_toin.record_compression( - tool_signature=signature, - original_count=50, - compressed_count=10, - original_tokens=5000, - compressed_tokens=1000, - strategy="smart_sample", - ) - - # Get initial strategy success rate - pattern = fresh_toin._patterns.get(signature.structure_hash) - assert pattern is not None, "Pattern should exist after compressions" - - initial_rate = pattern.strategy_success_rates.get("smart_sample", 1.0) - assert initial_rate > 0, "Initial success rate should be positive" - - # Simulate retrieval events (which indicate compression was too aggressive) - for _ in range(3): - fresh_toin.record_retrieval( - tool_signature_hash=signature.structure_hash, - retrieval_type="full", - query="test query", - query_fields=["id"], - strategy="smart_sample", - ) - - # Verify success rate decreased - final_rate = pattern.strategy_success_rates.get("smart_sample", 1.0) - assert final_rate < initial_rate, ( - f"Success rate should decrease after retrievals: {initial_rate} -> {final_rate}" - ) - - def test_full_feedback_loop(self, fresh_toin, fresh_store): - """Test the complete feedback loop: compress → retrieve → learn → recommend.""" - # Create test data - items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)] - content = json.dumps(items) - signature = ToolSignature.from_items(items) - - # Step 1: Compress with SmartCrusher (records to TOIN) - ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False) - crusher = SmartCrusher( - SmartCrusherConfig(max_items_after_crush=10, use_feedback_hints=True), - ccr_config=ccr_config, - ) - - # Multiple compressions to build pattern - for _ in range(5): - crusher._smart_crush_content(content, tool_name="test_tool") - - # Step 2: Verify TOIN has a pattern - pattern = fresh_toin._patterns.get(signature.structure_hash) - assert pattern is not None, "TOIN should have a pattern after compressions" - assert pattern.total_compressions >= 5, "Should have recorded 5 compressions" - - # Step 3: Simulate retrievals (indicating compression was too aggressive) - # Find the stored entry hash - entries = [entry for _, entry in fresh_store._backend.items()] - assert len(entries) > 0, "Should have cached entries" - - # Retrieve multiple times to trigger learning - for entry in entries[:3]: - fresh_store.retrieve(entry.hash, query="find all items") - - # Step 4: Verify TOIN learned from retrievals - pattern = fresh_toin._patterns.get(signature.structure_hash) - assert pattern is not None, "Pattern should exist after retrievals" - - # CRITICAL: Verify field-level learning actually happened - # This assertion would have caught the bug where compression_store - # wasn't passing retrieved_items to TOIN - assert len(pattern.field_semantics) > 0, ( - "TOIN should learn field semantics from retrieved items. " - "If this fails, the production code path (CompressionStore -> TOIN) is broken." - ) - - # Verify retrieval stats were updated - assert pattern.total_retrievals >= 1, "Should have recorded retrievals" - - recommendation = fresh_toin.get_recommendation(signature, "find all items") - - # After many retrievals, TOIN should recommend more items - # Default is 15-20, but with high retrieval rate it should go higher - assert recommendation.confidence > 0, "Recommendation should have confidence" - - def test_preserve_fields_used_in_compression(self, fresh_toin, fresh_store): - """Test that TOIN preserve_fields are used during compression planning.""" - # Create test data with specific fields - items = [{"id": i, "score": 100 - i, "category": f"cat_{i % 3}"} for i in range(50)] - signature = ToolSignature.from_items(items) - - # Record compressions and retrievals that query the "category" field - for _ in range(10): - fresh_toin.record_compression( - tool_signature=signature, - original_count=50, - compressed_count=10, - original_tokens=5000, - compressed_tokens=1000, - strategy="smart_sample", - ) - - # Record retrievals that query "category" - for _ in range(5): - fresh_toin.record_retrieval( - tool_signature_hash=signature.structure_hash, - retrieval_type="search", - query="category:cat_1", - query_fields=["category"], - strategy="smart_sample", - ) - - # Get recommendation - should now preserve "category" field - recommendation = fresh_toin.get_recommendation(signature, "find all items in cat_1") - - # Verify the recommendation reflects learning - assert recommendation.source in ("local", "network", "default"), ( - f"Should have a valid source: {recommendation.source}" - ) - - def test_instance_id_stable_across_restarts(self): - """Test that instance_id is stable across restarts.""" - reset_toin() - with tempfile.TemporaryDirectory() as tmpdir: - storage_path = str(Path(tmpdir) / "toin_stable.json") - - # Create first TOIN instance - toin1 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path)) - instance_id_1 = toin1._instance_id - - # Create second instance with same path (simulating restart) - toin2 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path)) - instance_id_2 = toin2._instance_id - - # Instance IDs should be the same since derived from path - assert instance_id_1 == instance_id_2, ( - f"Instance ID should be stable: {instance_id_1} vs {instance_id_2}" - ) - - def test_atomic_save(self): - """Test that save() uses atomic writes.""" - reset_toin() - with tempfile.TemporaryDirectory() as tmpdir: - storage_path = str(Path(tmpdir) / "toin_atomic.json") - toin = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path)) - - # Create some data - items = [{"id": i, "name": f"test_{i}"} for i in range(10)] - signature = ToolSignature.from_items(items) - - toin.record_compression( - tool_signature=signature, - original_count=10, - compressed_count=5, - original_tokens=1000, - compressed_tokens=500, - strategy="smart_sample", - ) - - # Save - toin.save() - - # Verify file exists and is valid JSON - saved_path = Path(storage_path) - assert saved_path.exists(), "Save file should exist" - - with open(saved_path) as f: - data = json.load(f) - - assert "patterns" in data, "Saved data should have patterns" - assert len(data["patterns"]) > 0, "Should have at least one pattern" - - def test_query_patterns_merged_on_import(self): - """Test that query patterns are merged when importing patterns.""" - reset_toin() - with tempfile.TemporaryDirectory() as tmpdir: - storage_path = str(Path(tmpdir) / "toin_merge.json") - toin = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path)) - - # Create local pattern - items = [{"id": i, "name": f"test_{i}"} for i in range(10)] - signature = ToolSignature.from_items(items) - - toin.record_compression( - tool_signature=signature, - original_count=10, - compressed_count=5, - original_tokens=1000, - compressed_tokens=500, - strategy="smart_sample", - ) - - # Record local query pattern - toin.record_retrieval( - tool_signature_hash=signature.structure_hash, - retrieval_type="search", - query="local_pattern:value", - query_fields=["local_pattern"], - strategy="smart_sample", - ) - - # Create import data with different query pattern - import_data = { - "patterns": { - signature.structure_hash: { - "tool_signature_hash": signature.structure_hash, - "total_compressions": 100, - "total_retrievals": 20, - "avg_original_count": 50, - "avg_compressed_count": 10, - "avg_compression_ratio": 0.2, - "retrieval_rate": 0.2, - "common_query_patterns": ["imported_pattern:x"], - "strategy_success_rates": {"smart_sample": 0.8}, - "preserve_fields": ["imported_field"], - "user_count": 50, - "last_updated": 1234567890.0, - } - } - } - - # Import patterns - toin.import_patterns(import_data) - - # Verify patterns were merged - pattern = toin._patterns.get(signature.structure_hash) - assert pattern is not None, "Pattern should exist after merge" - - # Check that both query patterns exist - assert "imported_pattern:x" in pattern.common_query_patterns, ( - "Imported query pattern should be present" - ) - - -class TestStoreToTOINHash: - """Test the hash correlation between compression_store and TOIN.""" - - def test_hash_matches_between_store_and_toin(self, fresh_toin, fresh_store): - """Test that the hash stored in compression_store matches TOIN events.""" - # Use 50 items with score field to ensure compression threshold is met - items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)] - content = json.dumps(items) - signature = ToolSignature.from_items(items) - - # Compress with aggressive settings to ensure items are actually reduced - ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False) - crusher = SmartCrusher( - SmartCrusherConfig(max_items_after_crush=10), - ccr_config=ccr_config, - ) - crushed, was_modified, info = crusher._smart_crush_content(content, tool_name="hash_test") - - # Verify compression actually happened - assert was_modified, f"Content should be modified by compression: {info}" - - # Get the stored hash - entries = [entry for _, entry in fresh_store._backend.items()] - assert len(entries) >= 1, ( - f"Should have stored entry. Modified: {was_modified}, Info: {info}" - ) - stored_hash = entries[0].tool_signature_hash - - # Verify it matches ToolSignature - assert stored_hash == signature.structure_hash, ( - f"Store hash {stored_hash} should match signature {signature.structure_hash}" - ) - - # Verify TOIN can receive events for this hash - fresh_toin.record_compression( - tool_signature=signature, - original_count=50, - compressed_count=10, - original_tokens=5000, - compressed_tokens=1000, - strategy="smart_sample", - ) - - pattern = fresh_toin._patterns.get(signature.structure_hash) - assert pattern is not None, "TOIN should have pattern for same hash" diff --git a/tests/test_transforms/test_anchor_selector.py b/tests/test_transforms/test_anchor_selector.py deleted file mode 100644 index aa73c4c70..000000000 --- a/tests/test_transforms/test_anchor_selector.py +++ /dev/null @@ -1,1186 +0,0 @@ -"""Tests for AnchorSelector - Adaptive position-based anchor allocation. - -Comprehensive tests covering: -- Adversarial positions: Important data NOT at expected positions -- Size adaptation: Anchor allocation scaling with array size -- Pattern-aware anchoring: Different strategies for different data patterns -- Query-aware anchoring: Query-based anchor adjustment -- Information density: Unique/informative item selection -- Coverage metrics: Distribution coverage verification -- Edge cases: Boundary conditions and special scenarios - -These tests verify the AnchorSelector replaces the static "first 3 + last 2" -preservation with adaptive, pattern-aware anchor selection. -""" - -import json - -import pytest - -from headroom import OpenAIProvider, SmartCrusherConfig, Tokenizer -from headroom.transforms.smart_crusher import ( - SmartAnalyzer, - SmartCrusher, -) - -# ============================================================================= -# Test Fixtures -# ============================================================================= - -_provider = OpenAIProvider() - - -def get_tokenizer(model: str = "gpt-4o") -> Tokenizer: - """Get a tokenizer for tests using OpenAI provider.""" - token_counter = _provider.get_token_counter(model) - return Tokenizer(token_counter, model) - - -@pytest.fixture -def tokenizer(): - """Provide a tokenizer for tests.""" - return get_tokenizer() - - -@pytest.fixture -def default_config(): - """Default SmartCrusherConfig for testing.""" - return SmartCrusherConfig( - enabled=True, - min_items_to_analyze=3, - min_tokens_to_crush=0, # Always crush for tests - max_items_after_crush=10, - variance_threshold=2.0, - ) - - -@pytest.fixture -def analyzer(default_config): - """SmartAnalyzer instance for testing.""" - return SmartAnalyzer(default_config) - - -@pytest.fixture -def crusher(default_config): - """SmartCrusher instance for testing.""" - return SmartCrusher(default_config) - - -# ============================================================================= -# Test Data Generators -# ============================================================================= - - -def generate_uniform_items(n: int, value: int = 0) -> list[dict]: - """Generate array of identical items.""" - return [{"id": "same", "value": value, "status": "ok"} for _ in range(n)] - - -def generate_numbered_items(n: int, with_value: bool = True) -> list[dict]: - """Generate items with sequential IDs for position tracking.""" - items = [] - for i in range(n): - item = {"id": i, "name": f"Item {i}"} - if with_value: - item["value"] = i * 10 - items.append(item) - return items - - -def generate_time_series_data( - n: int = 50, - spike_positions: list[int] | None = None, - spike_value: float = 1000.0, -) -> list[dict]: - """Generate time series with optional spikes at specific positions.""" - items = [] - for i in range(n): - value = 100.0 + (i * 0.5) # Slight upward trend - if spike_positions and i in spike_positions: - value = spike_value - items.append( - { - "timestamp": f"2025-01-{(i % 28) + 1:02d}T{(i % 24):02d}:00:00Z", - "value": value, - "metric": "cpu_usage", - } - ) - return items - - -def generate_log_data( - n: int = 50, - error_positions: list[int] | None = None, -) -> list[dict]: - """Generate log-style data with optional errors at specific positions.""" - levels = ["INFO", "DEBUG", "WARN"] - items = [] - for i in range(n): - level = levels[i % len(levels)] - if error_positions and i in error_positions: - level = "ERROR" - message = f"Critical failure at step {i}: connection timeout" - else: - message = f"Processing request {i} successfully" - items.append( - { - "level": level, - "message": message, - "timestamp": f"2025-01-06T{12 + (i // 60):02d}:{i % 60:02d}:00Z", - } - ) - return items - - -def generate_search_results(n: int = 50) -> list[dict]: - """Generate search results with scores (higher = better, at front).""" - return [ - { - "id": f"doc_{i}", - "title": f"Document {i}", - "score": 1.0 - (i * 0.02), # Scores decrease as index increases - "snippet": f"This is a snippet from document {i}...", - } - for i in range(n) - ] - - -def generate_categorized_items( - categories: dict[str, int], -) -> list[dict]: - """Generate items with specific category distribution. - - Args: - categories: Dict mapping category name to count, e.g. {"A": 30, "B": 30, "C": 40} - """ - items = [] - i = 0 - for category, count in categories.items(): - for _ in range(count): - items.append({"id": i, "category": category, "name": f"Item {i}"}) - i += 1 - return items - - -# ============================================================================= -# Helper Functions -# ============================================================================= - - -def crush_items( - items: list[dict], - tokenizer: Tokenizer, - max_items: int = 10, - query: str = "", - min_items: int = 3, -) -> list[dict]: - """Helper to crush items and return the result list. - - Args: - items: Array of items to compress. - tokenizer: Tokenizer instance. - max_items: Maximum items after crushing. - query: Optional query context. - min_items: Minimum items to analyze. - - Returns: - List of preserved items after crushing. - """ - messages = [ - {"role": "system", "content": "You are helpful."}, - ] - - if query: - messages.append({"role": "user", "content": query}) - messages.append( - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "search_items", "arguments": "{}"}, - } - ], - } - ) - - messages.append({"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}) - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=min_items, - max_items_after_crush=max_items, - ) - crusher = SmartCrusher(config) - result = crusher.apply(messages, tokenizer) - - # Parse result - tool_content = result.messages[-1]["content"] - json_part = tool_content.split("\n dict[str, list[int]]: - """Categorize preserved item positions into front/middle/back. - - Args: - result: List of preserved items (must have 'id' field with original index). - original_size: Size of original array. - - Returns: - Dict with 'front', 'middle', 'back' keys containing lists of preserved indices. - """ - front_boundary = int(original_size * 0.1) - back_boundary = int(original_size * 0.9) - - positions = {"front": [], "middle": [], "back": []} - - for item in result: - idx = item.get("id") - if idx is None: - continue - - if isinstance(idx, int): - if idx < front_boundary: - positions["front"].append(idx) - elif idx >= back_boundary: - positions["back"].append(idx) - else: - positions["middle"].append(idx) - - return positions - - -# ============================================================================= -# TestAdversarialPositions -# ============================================================================= - - -class TestAdversarialPositions: - """Test scenarios that break 'first 3 + last 2' assumption. - - These tests verify that important data is preserved regardless of position, - not just because it happens to be at the front or back of the array. - """ - - def test_important_data_in_middle(self, tokenizer): - """Critical error item at position 50 of 100-item array should be preserved. - - The error item is in the middle where static anchoring would miss it. - It should be preserved due to error detection, not position. - """ - items = [{"id": i, "status": "ok", "value": i * 10} for i in range(100)] - # Place critical error in the middle - items[50] = { - "id": 50, - "status": "error", - "error_code": "CRITICAL", - "message": "Connection failed", - } - - crushed = crush_items(items, tokenizer, max_items=15) - - # Error item MUST be preserved regardless of position - preserved_ids = [item["id"] for item in crushed] - assert 50 in preserved_ids, "Critical error at position 50 should be preserved" - - # Verify it's preserved for the right reason (has error indicator) - error_items = [item for item in crushed if item.get("status") == "error"] - assert len(error_items) >= 1, "At least one error item should be preserved" - - def test_spike_not_at_boundaries(self, tokenizer): - """Numeric spike at position 75 of 100-item array must be preserved. - - The spike represents an anomaly that should be detected statistically, - not missed because it's not at the front or back. - """ - items = generate_time_series_data(100, spike_positions=[75], spike_value=1000.0) - # Add id field for tracking - for i, item in enumerate(items): - item["id"] = i - - crushed = crush_items(items, tokenizer, max_items=15) - - # Spike MUST be preserved as anomaly - has_spike = any(item.get("value", 0) > 500 for item in crushed) - assert has_spike, "Anomalous spike at position 75 should be preserved" - - # Verify the specific position was kept - preserved_ids = [item.get("id") for item in crushed] - assert 75 in preserved_ids, "Position 75 with spike should be in result" - - def test_multiple_spikes_scattered(self, tokenizer): - """Multiple spikes at positions 25, 50, 75 should all be preserved.""" - spike_positions = [25, 50, 75] - items = generate_time_series_data(100, spike_positions=spike_positions, spike_value=999.0) - for i, item in enumerate(items): - item["id"] = i - - crushed = crush_items(items, tokenizer, max_items=15) - - # All spikes should be preserved - preserved_ids = {item.get("id") for item in crushed} - for pos in spike_positions: - assert pos in preserved_ids, f"Spike at position {pos} should be preserved" - - def test_first_items_identical(self, tokenizer): - """First 10 identical items - should NOT waste all anchor slots on them. - - When first N items are identical, we should keep at most 1-2 of them, - not waste 3 anchor slots on duplicates. - """ - # First 10 items are identical - identical_items = [{"id": "same", "value": 0, "type": "duplicate"} for _ in range(10)] - # Rest are unique - unique_items = [{"id": f"unique_{i}", "value": i * 10, "type": "unique"} for i in range(90)] - items = identical_items + unique_items - - crushed = crush_items(items, tokenizer, max_items=10) - - # Should NOT have multiple identical items - same_count = sum(1 for item in crushed if item.get("id") == "same") - assert same_count <= 2, f"Got {same_count} identical items, expected at most 2" - - # Should have some unique items - unique_count = sum(1 for item in crushed if item.get("type") == "unique") - assert unique_count >= 5, f"Expected at least 5 unique items, got {unique_count}" - - def test_last_items_identical(self, tokenizer): - """Last 10 identical items - should NOT waste anchor slots. - - Similar to above but for back anchors - when last N items are identical, - we should preserve variety instead of duplicates. - """ - # First 90 items are unique - unique_items = [{"id": f"unique_{i}", "value": i * 10, "type": "unique"} for i in range(90)] - # Last 10 items are identical - identical_items = [{"id": "same", "value": 100, "type": "duplicate"} for _ in range(10)] - items = unique_items + identical_items - - crushed = crush_items(items, tokenizer, max_items=10) - - # Should NOT have multiple identical items - same_count = sum(1 for item in crushed if item.get("id") == "same") - assert same_count <= 2, f"Got {same_count} identical items, expected at most 2" - - def test_relevant_item_in_middle(self, tokenizer): - """Item matching query in middle position must be found. - - When user queries for a specific item that's at position 42, - it should be preserved even though it's not at front/back. - """ - items = [{"id": i, "name": f"item_{i}", "status": "active"} for i in range(100)] - # Put target item in the middle - items[42]["name"] = "target_special_item" - items[42]["description"] = "This is what the user is looking for" - - crushed = crush_items( - items, - tokenizer, - max_items=10, - query="Find target_special_item", - ) - - # Query-matched item MUST be preserved - has_target = any("target_special_item" in item.get("name", "") for item in crushed) - assert has_target, "Item matching query should be preserved regardless of position" - - def test_error_in_middle_of_otherwise_uniform_data(self, tokenizer): - """Single error in middle of 100 identical OK items must be preserved.""" - items = [{"id": i, "status": "ok", "data": "normal"} for i in range(100)] - items[47] = {"id": 47, "status": "failed", "error": "Unexpected failure"} - - crushed = crush_items(items, tokenizer, max_items=15) - - # Error must be preserved - error_items = [item for item in crushed if item.get("status") == "failed"] - assert len(error_items) >= 1, "Error item at position 47 should be preserved" - assert error_items[0]["id"] == 47, "The specific error item should be preserved" - - -# ============================================================================= -# TestSizeAdaptation -# ============================================================================= - - -class TestSizeAdaptation: - """Test that anchor allocation scales with array size. - - Larger arrays should allocate proportionally more anchor slots - to ensure adequate coverage across the data range. - """ - - @pytest.mark.parametrize( - "size,max_items,expected_min_anchors", - [ - (20, 10, 3), # Small array: at least 3 anchors (front + back) - (100, 15, 4), # Medium array: at least 4 anchors - (500, 20, 5), # Large array: at least 5 anchors - (2000, 25, 6), # Very large: at least 6 anchors - ], - ) - def test_anchor_count_scales(self, tokenizer, size, max_items, expected_min_anchors): - """Anchor count should increase with array size. - - Verifies that the number of items from boundary regions (front 10%, back 10%) - increases as the array size grows. - """ - items = generate_numbered_items(size) - - crushed = crush_items(items, tokenizer, max_items=max_items) - - # Count items from first 10% and last 10% - front_boundary = int(size * 0.1) - back_boundary = int(size * 0.9) - - anchor_count = sum( - 1 for item in crushed if item["id"] < front_boundary or item["id"] >= back_boundary - ) - - assert anchor_count >= expected_min_anchors, ( - f"Array of size {size} should have at least {expected_min_anchors} " - f"anchors from boundary regions, got {anchor_count}" - ) - - def test_small_array_high_preservation(self, tokenizer): - """Small arrays (< max_items) should preserve most/all items. - - When the array is smaller than max_items, there's no need to drop items. - """ - items = generate_numbered_items(8) - - # max_items=20 is larger than array size - crushed = crush_items(items, tokenizer, max_items=20) - - # Should preserve all or nearly all items - assert len(crushed) >= 7, f"Small array should preserve most items, got {len(crushed)}" - - def test_small_array_exact_size(self, tokenizer): - """Array exactly at max_items should preserve all items.""" - items = generate_numbered_items(10) - - crushed = crush_items(items, tokenizer, max_items=10) - - # Should preserve all items - assert len(crushed) == 10, "Array at max_items limit should keep all items" - - def test_large_array_efficient_sampling(self, tokenizer): - """Large arrays should sample efficiently across all positions. - - A 500-item array crushed to 20 items should have representation - from front, middle, and back regions. - """ - items = generate_numbered_items(500) - - crushed = crush_items(items, tokenizer, max_items=20) - - positions = get_preserved_positions(crushed, 500) - - # Should have representation from all regions - has_front = len(positions["front"]) >= 1 - has_back = len(positions["back"]) >= 1 - # Middle representation is optional but preferred for large arrays - has_middle = len(positions["middle"]) >= 1 - - assert has_front, "Large array should preserve items from front" - assert has_back, "Large array should preserve items from back" - # Middle coverage is important for large arrays - assert has_middle, "Large array should have some middle representation" - - def test_very_large_array_coverage(self, tokenizer): - """Very large array (1000+) should have good position distribution.""" - items = generate_numbered_items(1000) - - crushed = crush_items(items, tokenizer, max_items=25) - - positions = get_preserved_positions(crushed, 1000) - - # Calculate coverage spread - all_positions = positions["front"] + positions["middle"] + positions["back"] - if len(all_positions) >= 2: - spread = max(all_positions) - min(all_positions) - # Should span at least 80% of the array - assert spread >= 800, f"Preserved items should span array, spread was {spread}" - - -# ============================================================================= -# TestPatternAwareAnchoring -# ============================================================================= - - -class TestPatternAwareAnchoring: - """Test pattern-specific anchor strategies. - - Different data patterns (search results, logs, time series) should - use different anchor weighting strategies. - """ - - def test_search_results_front_heavy(self, tokenizer): - """Search results with scores should preserve more from front (high scores). - - Search results are sorted by relevance score, so top items are most important. - """ - items = generate_search_results(100) - for i, item in enumerate(items): - item["idx"] = i # Track original position - - crushed = crush_items(items, tokenizer, max_items=12) - - # Count items by their original position - front_count = sum(1 for item in crushed if item.get("idx", 100) < 30) - back_count = sum(1 for item in crushed if item.get("idx", 0) >= 70) - - # For search results, front (high scores) should dominate - assert front_count > back_count, ( - f"Search results should preserve more front items (high scores), " - f"got front={front_count}, back={back_count}" - ) - - # Top results should definitely be present - ids = [item["id"] for item in crushed] - assert "doc_0" in ids, "Top search result should be preserved" - assert "doc_1" in ids, "Second search result should be preserved" - - def test_logs_back_heavy(self, tokenizer): - """Logs with timestamps should preserve more recent items (back of array). - - Logs are typically ordered chronologically, with recent items at the end. - """ - items = generate_log_data(100) - for i, item in enumerate(items): - item["idx"] = i - - crushed = crush_items(items, tokenizer, max_items=12) - - # For logs, back (recent) should be emphasized - # Extract indices - use 'idx' field we added - indices = [item.get("idx", 0) for item in crushed] - - recent_count = sum(1 for idx in indices if idx >= 70) - old_count = sum(1 for idx in indices if idx < 30) - - # Recent logs should be at least as represented as old logs - assert recent_count >= old_count, ( - f"Logs should preserve recent items, got recent={recent_count}, old={old_count}" - ) - - def test_time_series_balanced(self, tokenizer): - """Time series should have balanced front/back representation. - - For trend analysis, we need both the start and end of the time series. - """ - items = generate_time_series_data(100) - for i, item in enumerate(items): - item["id"] = i - - crushed = crush_items(items, tokenizer, max_items=12) - - positions = get_preserved_positions(crushed, 100) - - front_count = len(positions["front"]) - back_count = len(positions["back"]) - - # Should be relatively balanced for time series (within 2:1 ratio) - if front_count > 0 and back_count > 0: - ratio = max(front_count, back_count) / min(front_count, back_count) - assert ratio <= 3, f"Time series should have balanced anchors, ratio was {ratio}" - - def test_generic_distributed(self, tokenizer): - """Generic data should sample across all positions. - - When pattern is unknown, sampling should be distributed rather than - heavily weighted to any particular region. - """ - items = generate_numbered_items(100) - - crushed = crush_items(items, tokenizer, max_items=15) - - positions = get_preserved_positions(crushed, 100) - - # Should have items from multiple regions - regions_with_items = sum(1 for region in ["front", "middle", "back"] if positions[region]) - - assert regions_with_items >= 2, ( - f"Generic data should cover multiple regions, got {regions_with_items}" - ) - - -# ============================================================================= -# TestQueryAwareAnchoring -# ============================================================================= - - -class TestQueryAwareAnchoring: - """Test query-based anchor adjustment. - - User queries containing temporal keywords should shift anchor weighting. - """ - - def test_latest_query_shifts_to_back(self, tokenizer): - """'Latest' in query should preserve more recent items.""" - items = [{"id": i, "created": f"2024-01-{i:02d}"} for i in range(1, 31)] - - crushed = crush_items( - items, - tokenizer, - max_items=8, - query="Show me the latest entries", - ) - - ids = [item["id"] for item in crushed] - recent_count = sum(1 for id in ids if id > 20) - - # Should have multiple recent items due to "latest" keyword - assert recent_count >= 2, ( - f"Query with 'latest' should preserve recent items, got {recent_count}" - ) - - def test_recent_query_shifts_to_back(self, tokenizer): - """'Recent' in query should preserve more recent items.""" - items = generate_log_data(50) - for i, item in enumerate(items): - item["idx"] = i - - crushed = crush_items( - items, - tokenizer, - max_items=10, - query="Show me recent log entries", - ) - - indices = [item.get("idx", 0) for item in crushed] - recent_count = sum(1 for idx in indices if idx >= 35) - - assert recent_count >= 2, "Query with 'recent' should have multiple recent items" - - def test_first_query_shifts_to_front(self, tokenizer): - """'First' in query should preserve earlier items.""" - items = [{"id": i, "created": f"2024-01-{i:02d}"} for i in range(1, 31)] - - crushed = crush_items( - items, - tokenizer, - max_items=8, - query="Show me the first entries", - ) - - ids = [item["id"] for item in crushed] - early_count = sum(1 for id in ids if id < 10) - - # Should have multiple early items due to "first" keyword - assert early_count >= 2, ( - f"Query with 'first' should preserve early items, got {early_count}" - ) - - def test_oldest_query_shifts_to_front(self, tokenizer): - """'Oldest' in query should preserve earlier items.""" - items = generate_numbered_items(50) - - crushed = crush_items( - items, - tokenizer, - max_items=10, - query="Find the oldest records", - ) - - ids = [item["id"] for item in crushed] - early_count = sum(1 for id in ids if id < 15) - - assert early_count >= 3, "Query with 'oldest' should have multiple early items" - - def test_specific_id_query_finds_item(self, tokenizer): - """Query for specific ID should find it regardless of position.""" - items = [{"id": f"item_{i:04d}", "value": i} for i in range(200)] - - crushed = crush_items( - items, - tokenizer, - max_items=10, - query="Find item_0123", - ) - - # Item at position 123 should be found - ids = [item["id"] for item in crushed] - assert "item_0123" in ids, "Specific ID query should find the item" - - def test_no_query_uses_default_weights(self, tokenizer): - """Without query, use pattern-based defaults.""" - items = generate_numbered_items(100) - - crushed = crush_items(items, tokenizer, max_items=15) - - # Without query, should use default anchoring (some front, some back) - positions = get_preserved_positions(crushed, 100) - - assert len(positions["front"]) >= 1, "Default should include front items" - assert len(positions["back"]) >= 1, "Default should include back items" - - -# ============================================================================= -# TestInformationDensity -# ============================================================================= - - -class TestInformationDensity: - """Test information-density based selection. - - Items with unique or rare properties should be preferred over common ones. - """ - - def test_unique_items_preferred(self, tokenizer): - """Items with rare field values should be preferred over common ones. - - When most items have status="ok" but a few have status="warning", - the warning items should be preserved as they're more informative. - """ - items = [] - for i in range(100): - status = "warning" if i in [25, 50, 75] else "ok" - items.append({"id": i, "status": status, "value": i}) - - crushed = crush_items(items, tokenizer, max_items=15) - - # Warning items (rare status) should be preserved - warning_count = sum(1 for item in crushed if item.get("status") == "warning") - assert warning_count >= 2, f"Rare status items should be preferred, got {warning_count}" - - def test_dedup_identical_items(self, tokenizer): - """Identical items should be deduplicated in anchor selection. - - If positions 0-5 all have identical content, we shouldn't keep all of them. - """ - # First 6 items are identical - items = [{"id": "dup", "value": 0, "constant": "same"} for _ in range(6)] - # Rest are unique - items.extend([{"id": f"uniq_{i}", "value": i * 10, "unique": True} for i in range(94)]) - - crushed = crush_items(items, tokenizer, max_items=12) - - # Should not have many identical items - dup_count = sum(1 for item in crushed if item.get("id") == "dup") - unique_count = sum(1 for item in crushed if item.get("unique")) - - assert dup_count <= 2, f"Should deduplicate identical items, got {dup_count}" - assert unique_count >= 6, f"Should prefer unique items, got {unique_count}" - - def test_structural_outliers_preferred(self, tokenizer): - """Items with different structure should be preferred. - - An item with extra fields (like an error with stack trace) should be - preferred over uniform items. - """ - items = [{"id": i, "status": "ok"} for i in range(100)] - # Add a structurally different item in the middle - items[42] = { - "id": 42, - "status": "error", - "error_message": "Something went wrong", - "stack_trace": "at line 123...", - "error_code": "ERR_001", - } - - crushed = crush_items(items, tokenizer, max_items=15) - - # Structural outlier should be preserved - outlier = [item for item in crushed if item.get("stack_trace")] - assert len(outlier) >= 1, "Structurally different item should be preserved" - - def test_diverse_values_over_uniform(self, tokenizer): - """When selecting from candidates, prefer diverse values.""" - items = [] - # Create items with varying diversity - for i in range(100): - items.append( - { - "id": i, - "type": "common" if i < 90 else f"rare_type_{i}", - "value": i, - } - ) - - crushed = crush_items(items, tokenizer, max_items=15) - - # Should have some rare types - rare_types = [item for item in crushed if "rare" in item.get("type", "")] - assert len(rare_types) >= 1, "Rare type values should be preserved" - - -# ============================================================================= -# TestCoverageMetrics -# ============================================================================= - - -class TestCoverageMetrics: - """Test that preserved items represent the full distribution. - - Compression should maintain coverage of value ranges, categories, and time. - """ - - def test_value_range_coverage(self, tokenizer): - """Preserved items should cover the value range.""" - items = [{"id": i, "value": i} for i in range(100)] - - crushed = crush_items(items, tokenizer, max_items=12) - - values = [item["value"] for item in crushed] - - # Should cover most of the range [0, 100] - assert min(values) < 10, "Should have low values" - assert max(values) > 90, "Should have high values" - - # Should have some middle values too - middle_count = sum(1 for v in values if 30 < v < 70) - assert middle_count >= 1, "Should have some middle-range values" - - def test_category_coverage(self, tokenizer): - """Preserved items should represent multiple categories.""" - items = generate_categorized_items({"A": 30, "B": 30, "C": 40}) - - crushed = crush_items(items, tokenizer, max_items=12) - - categories = {item["category"] for item in crushed} - - # Should have at least 2 of 3 categories represented - assert len(categories) >= 2, f"Should cover multiple categories, got {categories}" - - def test_category_proportional_representation(self, tokenizer): - """Category distribution should roughly reflect original proportions.""" - items = generate_categorized_items({"major": 80, "minor": 20}) - - crushed = crush_items(items, tokenizer, max_items=15) - - major_count = sum(1 for item in crushed if item["category"] == "major") - minor_count = sum(1 for item in crushed if item["category"] == "minor") - - # Major category should have more items, but minor should be represented - assert major_count > minor_count, "Major category should dominate" - assert minor_count >= 1, "Minor category should still be represented" - - def test_temporal_coverage(self, tokenizer): - """Preserved items should span the time range.""" - # Create items spanning 12 months - items = [ - {"id": i, "timestamp": f"2024-{(i % 12) + 1:02d}-15", "event": f"event_{i}"} - for i in range(60) - ] - - crushed = crush_items(items, tokenizer, max_items=12) - - months = [int(item["timestamp"][5:7]) for item in crushed] - - # Should span a significant portion of the year - month_range = max(months) - min(months) - assert month_range >= 5, f"Should span multiple months, got range of {month_range}" - - def test_numeric_distribution_coverage(self, tokenizer): - """Preserved numeric values should represent the distribution.""" - # Create items with bimodal distribution - items = [] - for i in range(50): - items.append({"id": i, "value": 10 + (i % 5)}) # Low cluster: 10-15 - for i in range(50): - items.append({"id": 50 + i, "value": 90 + (i % 5)}) # High cluster: 90-95 - - crushed = crush_items(items, tokenizer, max_items=12) - - values = [item["value"] for item in crushed] - - # Should have items from both clusters - low_cluster = [v for v in values if v < 20] - high_cluster = [v for v in values if v > 85] - - assert len(low_cluster) >= 1, "Should have items from low cluster" - assert len(high_cluster) >= 1, "Should have items from high cluster" - - -# ============================================================================= -# TestEdgeCases -# ============================================================================= - - -class TestEdgeCases: - """Edge cases and boundary conditions.""" - - def test_empty_array(self, tokenizer): - """Empty array should return empty result.""" - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": "[]"}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=3, - max_items_after_crush=10, - ) - crusher = SmartCrusher(config) - result = crusher.apply(messages, tokenizer) - - tool_content = result.messages[-1]["content"] - json_part = tool_content.split("\n array size should return all items.""" - items = generate_numbered_items(8) - - crushed = crush_items(items, tokenizer, max_items=20) - - assert len(crushed) == 8, "Should return all items when max_items > array size" - - def test_max_items_equals_array(self, tokenizer): - """max_items == array size should return all items.""" - items = generate_numbered_items(15) - - crushed = crush_items(items, tokenizer, max_items=15) - - assert len(crushed) == 15, "Should return all items when max_items == array size" - - def test_all_items_have_errors(self, tokenizer): - """When all items have errors, quality guarantee preserves all. - - Current behavior: Error preservation takes precedence over max_items. - All error items are preserved to avoid losing critical information. - This is a deliberate design choice documented in SmartCrusher. - - NOTE: Future AnchorSelector may implement error deduplication or sampling - for cases where all items are errors, but that requires careful design. - """ - items = [{"id": i, "status": "error", "error_code": f"ERR_{i}"} for i in range(50)] - - crushed = crush_items(items, tokenizer, max_items=15) - - # Current behavior: all errors are preserved (quality guarantee) - # This is intentional - we don't want to lose error information - assert len(crushed) == 50, "All error items should be preserved (quality guarantee)" - - def test_none_values_handled(self, tokenizer): - """Items with None values should be handled gracefully.""" - items = [ - {"id": i, "value": None if i % 3 == 0 else i * 10, "name": f"Item {i}"} - for i in range(30) - ] - - crushed = crush_items(items, tokenizer, max_items=10) - - # Should not crash, should return valid items - assert len(crushed) > 0, "Should handle None values" - assert all(isinstance(item, dict) for item in crushed) - - def test_mixed_types_in_array(self, tokenizer): - """Array with mixed item structures should be handled.""" - items = [ - {"id": 0, "simple": True}, - {"id": 1, "nested": {"deep": {"value": 42}}}, - {"id": 2, "list_field": [1, 2, 3]}, - {"id": 3, "mixed": {"a": [1, 2], "b": "text"}}, - ] - # Add more simple items to trigger crushing - items.extend([{"id": i, "simple": True} for i in range(4, 20)]) - - crushed = crush_items(items, tokenizer, max_items=8, min_items=3) - - # Should preserve structurally interesting items - assert len(crushed) > 0, "Should handle mixed structures" - - def test_unicode_content_preserved(self, tokenizer): - """Unicode content should be preserved correctly.""" - # Use the full Unicode codepoint for rocket emoji (U+1F680) - rocket_emoji = "\U0001f680" # Full codepoint, not surrogate pair - items = [ - {"id": i, "name": f"Item {i} - \u4e2d\u6587 \u65e5\u672c\u8a9e {rocket_emoji}"} - for i in range(20) - ] - - crushed = crush_items(items, tokenizer, max_items=10) - - # Unicode should be preserved - for item in crushed: - assert "\u4e2d\u6587" in item["name"], "Chinese characters should be preserved" - assert rocket_emoji in item["name"], "Emoji should be preserved" - - def test_very_long_string_values(self, tokenizer): - """Items with very long string values should be handled.""" - items = [{"id": i, "data": "x" * 10000 if i == 10 else "short"} for i in range(30)] - - crushed = crush_items(items, tokenizer, max_items=10) - - # Should not crash - assert len(crushed) > 0, "Should handle long strings" - - def test_deeply_nested_items(self, tokenizer): - """Deeply nested items should be handled without stack overflow.""" - - def create_nested(depth: int) -> dict: - if depth == 0: - return {"value": "leaf"} - return {"nested": create_nested(depth - 1)} - - items = [{"id": i, "deep": create_nested(10)} for i in range(20)] - - crushed = crush_items(items, tokenizer, max_items=10) - - assert len(crushed) > 0, "Should handle deeply nested items" - - -# ============================================================================= -# TestAnchorConfigBehavior -# ============================================================================= - - -class TestAnchorConfigBehavior: - """Test configuration-driven anchor behavior.""" - - def test_high_max_items_reduces_compression(self, tokenizer): - """Higher max_items should preserve more items. - - NOTE: Data must have "importance signals" (errors, anomalies) to trigger - compression. Generic unique items without signals are SKIPPED by the - crushability analysis (conservative behavior to avoid losing entities). - - This test uses search results which always compress based on score. - """ - # Use search results - they always trigger TOP_N compression - items = generate_search_results(100) - - crushed_low = crush_items(items, tokenizer, max_items=8) - crushed_high = crush_items(items, tokenizer, max_items=25) - - assert len(crushed_high) > len(crushed_low), ( - f"Higher max_items should preserve more items, " - f"got low={len(crushed_low)}, high={len(crushed_high)}" - ) - - def test_min_items_to_analyze_threshold(self, tokenizer): - """Arrays below min_items_to_analyze should not be crushed.""" - items = generate_numbered_items(5) - - # Set min_items_to_analyze higher than array size - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=10, # Higher than array size - max_items_after_crush=3, - ) - crusher = SmartCrusher(config) - result = crusher.apply(messages, tokenizer) - - tool_content = result.messages[-1]["content"] - json_part = tool_content.split("\n= 1, "Items near change point should be preserved" diff --git a/tests/test_transforms/test_smart_crusher.py b/tests/test_transforms/test_smart_crusher.py deleted file mode 100644 index 238690e7d..000000000 --- a/tests/test_transforms/test_smart_crusher.py +++ /dev/null @@ -1,1359 +0,0 @@ -"""Tests for SmartCrusher transform. - -Comprehensive tests covering: -- SmartAnalyzer: Statistical analysis of arrays -- SmartCrusher: Intelligent compression with Safe V1 Recipe -- RelevanceScoring: Context extraction and item matching -- Edge cases: Malformed JSON, nested arrays, different message formats -""" - -import json - -import pytest - -from headroom import ( - OpenAIProvider, - RelevanceScorerConfig, - SmartCrusherConfig, - Tokenizer, -) -from headroom.relevance import RelevanceScore, RelevanceScorer -from headroom.transforms.smart_crusher import ( - CompressionStrategy, - SmartAnalyzer, - SmartCrusher, -) - -# ============================================================================= -# Test Fixtures -# ============================================================================= - -# Create a shared provider for tests -_provider = OpenAIProvider() - - -def get_tokenizer(model: str = "gpt-4o") -> Tokenizer: - """Get a tokenizer for tests using OpenAI provider.""" - token_counter = _provider.get_token_counter(model) - return Tokenizer(token_counter, model) - - -@pytest.fixture -def tokenizer(): - """Provide a tokenizer for tests.""" - return get_tokenizer() - - -@pytest.fixture -def default_config(): - """Default SmartCrusherConfig for testing.""" - return SmartCrusherConfig( - enabled=True, - min_items_to_analyze=3, - min_tokens_to_crush=0, # Always crush for tests - max_items_after_crush=10, - variance_threshold=2.0, - ) - - -@pytest.fixture -def analyzer(default_config): - """SmartAnalyzer instance for testing.""" - return SmartAnalyzer(default_config) - - -@pytest.fixture -def crusher(default_config): - """SmartCrusher instance for testing.""" - return SmartCrusher(default_config) - - -# ============================================================================= -# Test Data Generators -# ============================================================================= - - -def generate_time_series_data(n: int = 20, with_spike: bool = False) -> list[dict]: - """Generate time series data with optional anomaly.""" - data = [] - for i in range(n): - value = 100.0 + (i * 0.5) # Slight upward trend - if with_spike and i == n // 2: - value = 500.0 # Spike in the middle - data.append( - { - "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z", - "value": value, - "metric": "cpu_usage", - } - ) - return data - - -def generate_log_data(n: int = 20, with_errors: bool = False) -> list[dict]: - """Generate log-style data with optional errors.""" - data = [] - levels = ["INFO", "DEBUG", "WARN"] - for i in range(n): - level = levels[i % len(levels)] - if with_errors and i in [5, 15]: - level = "ERROR" - message = f"Connection failed: timeout after 30s (attempt {i})" - else: - message = f"Processing request {i} successfully" - data.append( - { - "level": level, - "message": message, - "timestamp": f"2025-01-06T{12 + (i // 60):02d}:{i % 60:02d}:00Z", - } - ) - return data - - -def generate_search_results(n: int = 20) -> list[dict]: - """Generate search results with scores.""" - return [ - { - "id": f"doc_{i}", - "title": f"Document {i}", - "score": 1.0 - (i * 0.05), - "snippet": f"This is a snippet from document {i}...", - } - for i in range(n) - ] - - -def generate_generic_data( - n: int = 20, - constant_field: bool = False, - with_signals: bool = False, -) -> list[dict]: - """Generate generic array data. - - Args: - n: Number of items to generate - constant_field: If True, type field is constant "product" - with_signals: If True, adds importance signals (errors, anomalies) - to enable crushing with new statistical detection - """ - items = [] - for i in range(n): - item = { - "id": i, - "name": f"Item {i}", - "type": "product" if constant_field else f"type_{i % 3}", - "active": True if constant_field else (i % 2 == 0), - } - if with_signals: - item["value"] = 100.0 - # Add some errors - if i == n // 4: - item["error"] = f"Error at {i}" - # Add some anomalies - if i == n // 2: - item["value"] = 99999.0 - items.append(item) - return items - - -# ============================================================================= -# TestSmartAnalyzer -# ============================================================================= - - -class TestSmartAnalyzer: - """Tests for SmartAnalyzer class.""" - - def test_analyze_empty_array(self, analyzer): - """Empty array should return analysis with no field stats.""" - result = analyzer.analyze_array([]) - - assert result.item_count == 0 - assert result.field_stats == {} - assert result.detected_pattern == "generic" - assert result.recommended_strategy == CompressionStrategy.NONE - assert result.constant_fields == {} - - def test_analyze_single_item(self, analyzer): - """Single item array should return analysis but no compression.""" - items = [{"id": 1, "name": "Test"}] - result = analyzer.analyze_array(items) - - assert result.item_count == 1 - assert "id" in result.field_stats - assert "name" in result.field_stats - # Single item means constant fields - assert result.field_stats["id"].is_constant - assert result.field_stats["name"].is_constant - - def test_analyze_numeric_field_stats(self, analyzer): - """Numeric fields should have correct statistics computed.""" - items = [ - {"value": 10.0}, - {"value": 20.0}, - {"value": 30.0}, - {"value": 40.0}, - {"value": 50.0}, - ] - result = analyzer.analyze_array(items) - - stats = result.field_stats["value"] - assert stats.field_type == "numeric" - assert stats.min_val == 10.0 - assert stats.max_val == 50.0 - assert stats.mean_val == 30.0 - assert stats.variance is not None - assert stats.variance > 0 - - def test_analyze_string_field_stats(self, analyzer): - """String fields should have correct statistics computed.""" - items = [ - {"name": "Alice"}, - {"name": "Bob"}, - {"name": "Alice"}, # Duplicate - {"name": "Charlie"}, - {"name": "Alice"}, # Another duplicate - ] - result = analyzer.analyze_array(items) - - stats = result.field_stats["name"] - assert stats.field_type == "string" - assert stats.avg_length is not None - assert stats.top_values is not None - # Alice appears 3 times, should be top - assert stats.top_values[0][0] == "Alice" - assert stats.top_values[0][1] == 3 - - def test_detect_time_series_pattern(self, analyzer): - """Time series data should be detected correctly.""" - # Create data with timestamp and numeric variance - # Include anomaly to provide an importance signal for crushing - items = [] - for i in range(40): - # Create variance-inducing data - value = 100.0 + (i * 2.0) # Steady increase with variance - if i == 20: - value = 999.0 # Anomaly provides importance signal - items.append( - { - "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z", - "value": value, - "metric": "cpu_usage", - } - ) - - result = analyzer.analyze_array(items) - - # Pattern should be detected as time_series (timestamp + numeric variance) - assert result.detected_pattern == "time_series" - # With anomaly signal, strategy should allow crushing - assert result.recommended_strategy in [ - CompressionStrategy.TIME_SERIES, - CompressionStrategy.SMART_SAMPLE, - ] - - def test_detect_time_series_pattern_with_change_points(self): - """Time series with clear change points should use TIME_SERIES strategy.""" - # The change point detection threshold is variance_threshold * std - # To detect a change point, the before/after mean difference must exceed this - # With bimodal data, std is very high. We need a lower variance_threshold - # to reliably detect change points, OR the test should use a config - # with lower variance threshold. - - config = SmartCrusherConfig( - min_items_to_analyze=3, - variance_threshold=1.0, # Lower threshold to detect changes - ) - analyzer = SmartAnalyzer(config) - - # Create data with clear step change - items = [] - for i in range(40): - if i < 20: - value = 100.0 + (i * 0.5) # Values around 100-110 - else: - value = 300.0 + ((i - 20) * 0.5) # Values around 300-310 (jump) - items.append( - { - "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z", - "value": value, - "metric": "cpu_usage", - } - ) - - result = analyzer.analyze_array(items) - - assert result.detected_pattern == "time_series" - # With lower variance_threshold, change points should be detected - value_stats = result.field_stats.get("value") - assert value_stats is not None - # Even with low threshold, bimodal data has high std - # The test verifies the strategy selection logic - if len(value_stats.change_points) > 0: - assert result.recommended_strategy == CompressionStrategy.TIME_SERIES - else: - # If change points still not detected, strategy falls back - assert result.recommended_strategy in [ - CompressionStrategy.TIME_SERIES, - CompressionStrategy.SMART_SAMPLE, - ] - - def test_detect_logs_pattern(self, analyzer): - """Log data should be detected correctly.""" - # Use logs WITH errors to provide importance signal - items = generate_log_data(20, with_errors=True) - result = analyzer.analyze_array(items) - - # With structural detection, logs are detected as logs pattern - # but strategy depends on crushability analysis - assert result.detected_pattern in ["logs", "generic"] - # With error items providing signal, crushing can proceed - assert result.recommended_strategy in [ - CompressionStrategy.CLUSTER_SAMPLE, - CompressionStrategy.SMART_SAMPLE, - CompressionStrategy.SKIP, # May still skip if other conditions met - ] - - def test_detect_search_results_pattern(self, analyzer): - """Search results with scores should be detected correctly.""" - items = generate_search_results(20) - result = analyzer.analyze_array(items) - - assert result.detected_pattern == "search_results" - assert result.recommended_strategy == CompressionStrategy.TOP_N - - def test_detect_generic_pattern(self, analyzer): - """Generic data without special patterns should be detected.""" - items = generate_generic_data(20) - result = analyzer.analyze_array(items) - - assert result.detected_pattern == "generic" - # With new crushability analysis: unique IDs + no importance signal = SKIP - # This is the safe behavior to avoid dropping important unique entities - assert result.recommended_strategy in [ - CompressionStrategy.SMART_SAMPLE, - CompressionStrategy.SKIP, # More conservative when no signal present - ] - - def test_detect_change_points(self, analyzer): - """Change points should be detected in numeric data with variance.""" - # Create data with clear change point - items = [] - for i in range(30): - if i < 15: - value = 100.0 + (i * 0.1) # Low values - else: - value = 200.0 + ((i - 15) * 0.1) # High values after change - items.append({"timestamp": f"2025-01-{(i % 28) + 1:02d}", "metric": value}) - - result = analyzer.analyze_array(items) - - # Should detect change point around index 15 - metric_stats = result.field_stats.get("metric") - assert metric_stats is not None - assert metric_stats.change_points is not None - # Change points should be near the transition - if metric_stats.change_points: - assert any(10 <= cp <= 20 for cp in metric_stats.change_points) - - def test_constant_field_detection(self, analyzer): - """Constant fields should be identified.""" - items = generate_generic_data(20, constant_field=True) - result = analyzer.analyze_array(items) - - # type field should be constant ("product") - type_stats = result.field_stats.get("type") - assert type_stats is not None - assert type_stats.is_constant - assert type_stats.constant_value == "product" - - # Constant fields should be in constant_fields dict - assert "type" in result.constant_fields - assert result.constant_fields["type"] == "product" - - -# ============================================================================= -# TestSmartCrusher -# ============================================================================= - - -class TestSmartCrusher: - """Tests for SmartCrusher transform.""" - - def test_should_apply_below_threshold(self, tokenizer): - """Should not apply when tokens below min_tokens_to_crush.""" - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": '{"status": "ok"}'}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=1000, # High threshold - ) - crusher = SmartCrusher(config) - - assert not crusher.should_apply(messages, tokenizer) - - def test_should_apply_no_arrays(self, tokenizer): - """Should not apply when no crushable arrays present.""" - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": '{"status": "ok", "value": 123}'}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - ) - crusher = SmartCrusher(config) - - assert not crusher.should_apply(messages, tokenizer) - - def test_should_apply_small_array(self, tokenizer): - """Should not apply when array is below min_items_to_analyze.""" - small_array = [{"id": i} for i in range(3)] - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(small_array)}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=10, # Array too small - ) - crusher = SmartCrusher(config) - - assert not crusher.should_apply(messages, tokenizer) - - def test_crush_time_series_keeps_change_points(self, tokenizer, default_config): - """Time series crushing should preserve items around change points.""" - # Create data with clear change point AND an anomaly signal - items = [] - for i in range(30): - if i < 15: - value = 100.0 - else: - value = 200.0 # Jump at index 15 - # Add anomaly to provide importance signal for crushing - if i == 25: - value = 999.0 # Extreme anomaly - items.append( - { - "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z", - "value": value, - } - ) - - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=3, - max_items_after_crush=15, - preserve_change_points=True, - ) - crusher = SmartCrusher(config) - - result = crusher.apply(messages, tokenizer) - - # Parse result - tool_content = result.messages[1]["content"] - # Remove digest marker - json_part = tool_content.split("\n 2 std from mean).""" - items = [] - for i in range(30): - value = 100.0 + (i * 0.1) # Normal range ~100-103 - items.append({"id": i, "metric": value}) - - # Add anomaly in the middle - items[15]["metric"] = 500.0 # Way above mean - - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=3, - max_items_after_crush=10, - variance_threshold=2.0, - ) - crusher = SmartCrusher(config) - - result = crusher.apply(messages, tokenizer) - - # Parse result - tool_content = result.messages[1]["content"] - json_part = tool_content.split("\n RelevanceScore: - if '"id": 5' in item or '"id":5' in item: - return RelevanceScore(score=0.9, reason="mock high score") - return RelevanceScore(score=0.0, reason="mock low score") - - def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]: - return [self.score(item, context) for item in items] - - items = generate_generic_data(30) - - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "user", "content": "Find the special item"}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=3, - max_items_after_crush=10, - ) - crusher = SmartCrusher(config, scorer=MockScorer()) - - result = crusher.apply(messages, tokenizer) - - # Parse result - tool_content = result.messages[-1]["content"] - json_part = tool_content.split("\n 0 - assert any("smart" in t.lower() for t in result.transforms_applied) - - def test_token_reduction(self, tokenizer): - """Token count should be reduced after crushing.""" - items = generate_generic_data(100) - - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}, - ] - - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=3, - max_items_after_crush=10, - ) - crusher = SmartCrusher(config) - - result = crusher.apply(messages, tokenizer) - - # Tokens should be reduced - assert result.tokens_after < result.tokens_before - - -# ============================================================================= -# Integration Tests -# ============================================================================= - - -class TestSmartCrusherIntegration: - """Integration tests for SmartCrusher with realistic scenarios.""" - - def test_database_query_results(self, tokenizer): - """Simulate crushing database query results.""" - # Simulate a database query returning many rows - items = [ - { - "user_id": f"usr_{i:05d}", - "email": f"user{i}@example.com", - "created_at": f"2025-01-{(i % 28) + 1:02d}T00:00:00Z", - "status": "active" if i % 10 != 0 else "inactive", - "login_count": i * 5, - } - for i in range(100) - ] - - messages = [ - {"role": "system", "content": "You are a database assistant."}, - {"role": "user", "content": "Show me users with email containing 'user50'"}, - { - "role": "assistant", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "query_users", "arguments": "{}"}, - } - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)}, - ] - - relevance_config = RelevanceScorerConfig(tier="bm25", relevance_threshold=0.1) - config = SmartCrusherConfig( - enabled=True, - min_tokens_to_crush=0, - min_items_to_analyze=3, - max_items_after_crush=15, - ) - crusher = SmartCrusher(config, relevance_config=relevance_config) - - result = crusher.apply(messages, tokenizer) - - # Parse result - tool_content = result.messages[3]["content"] - json_part = tool_content.split("\n SmartCrusher: + """Build a SmartCrusher with deterministic small-K config for tests.""" config = SmartCrusherConfig( enabled=True, min_items_to_analyze=min_items, @@ -30,102 +31,13 @@ def _make_crusher(max_items: int = 10, min_items: int = 3) -> SmartCrusher: return SmartCrusher(config=config) -# --------------------------------------------------------------------------- -# Bug 1: Number array type mixing -# --------------------------------------------------------------------------- - - -class TestNumberArraySchemaPreservation: - """_crush_number_array must return only original numeric values. - - Previously it prepended a stats summary string, producing - [string, int, int, ...] which violates the schema-preserving - guarantee and breaks type-aware JSON consumers. - """ - - def test_crushed_number_array_contains_only_numbers(self) -> None: - """Every element of the crushed array must be int or float.""" - crusher = _make_crusher(max_items=10) - numbers = list(range(50)) # 0..49, well above the n<=8 passthrough - crushed, strategy = crusher._crush_number_array(numbers) - - for i, item in enumerate(crushed): - assert isinstance(item, int | float), ( - f"Item {i} is {type(item).__name__} = {item!r}, expected int/float. " - f"Schema-preserving guarantee violated." - ) - - def test_crushed_number_array_subset_of_original(self) -> None: - """Every value in the crushed array must exist in the original.""" - crusher = _make_crusher(max_items=10) - numbers = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120] - crushed, _ = crusher._crush_number_array(numbers) - - original_set = set(numbers) - for item in crushed: - assert item in original_set, ( - f"Value {item!r} not in original array — generated content detected" - ) - - def test_stats_summary_in_strategy_not_in_array(self) -> None: - """Statistics should be communicated via strategy string, not array content.""" - crusher = _make_crusher(max_items=5) - numbers = list(range(100)) - crushed, strategy = crusher._crush_number_array(numbers) - - # Strategy should contain stats info - assert "number:" in strategy - - # Array should not contain any strings - strings_in_result = [x for x in crushed if isinstance(x, str)] - assert strings_in_result == [], f"Found string(s) in numeric array: {strings_in_result}" - - def test_number_array_passthrough_for_small(self) -> None: - """Arrays with n <= 8 should pass through unchanged.""" - crusher = _make_crusher() - small = [1, 2, 3, 4, 5] - crushed, strategy = crusher._crush_number_array(small) - assert crushed == small - assert strategy == "number:passthrough" - - def test_number_array_preserves_outliers(self) -> None: - """Outlier values should be preserved in the crushed output.""" - crusher = _make_crusher(max_items=10) - # Normal range + extreme outlier - numbers = [10] * 20 + [10000] - crushed, strategy = crusher._crush_number_array(numbers) - assert 10000 in crushed, "Outlier value 10000 was dropped" - - def test_number_array_preserves_boundaries(self) -> None: - """First and last values should always be kept.""" - crusher = _make_crusher(max_items=5) - numbers = list(range(100)) - crushed, strategy = crusher._crush_number_array(numbers) - assert crushed[0] == 0, "First value not preserved" - assert numbers[-1] in crushed, "Last value not preserved" - - def test_non_finite_passthrough(self) -> None: - """All-NaN/Inf arrays should return unchanged.""" - crusher = _make_crusher() - nans = [float("nan")] * 10 - crushed, strategy = crusher._crush_number_array(nans) - assert strategy == "number:no_finite" - assert len(crushed) == 10 - - def test_full_crush_pipeline_number_array_types(self) -> None: - """End-to-end: crushing a JSON number array via the public API.""" - crusher = _make_crusher(max_items=10) - content = json.dumps(list(range(50))) - result, was_modified, info = crusher._smart_crush_content(content) - - if was_modified: - parsed = json.loads(result) - assert isinstance(parsed, list) - for item in parsed: - assert isinstance(item, int | float), ( - f"Public API returned non-numeric item {item!r} in number array" - ) - +# Bug #1 (number array schema preservation) — invariant pinned by the +# Rust port (`crates/headroom-core/src/transforms/smart_crusher/crushers.rs:: +# crush_number_array` + its unit tests) and the parity fixtures +# (`tests/parity/fixtures/smart_crusher/number_array_40_changepoint*`). +# The Python `_crush_number_array` helper that the previous tests +# probed was removed when the Python implementation was retired in +# Stage 3c.1b. # --------------------------------------------------------------------------- # Bug 2: Race condition on _current_field_semantics @@ -212,79 +124,12 @@ class TestRecursionDepthLimit: assert isinstance(parsed, list) -# --------------------------------------------------------------------------- -# Stage 3c.1 lockstep bug fixes (#1 percentile, #2 sequential, #3 rare-status, -# #4 k-split). Each test pins the Python behavior post-fix; Rust has matching -# tests so parity fixtures byte-equal both languages. -# --------------------------------------------------------------------------- - - -class TestStage3c1BugFixes: - """Bugs fixed in lockstep with the Rust port at Stage 3c.1.""" - - # Bug #1 — percentile off-by-one (cosmetic, strategy string only). - def test_bug1_percentile_uses_linear_interpolation(self) -> None: - from headroom.transforms.smart_crusher import _percentile_linear - - # n=10 [10..100]: p25 index = 0.25 * 9 = 2.25 → 30*0.75 + 40*0.25 = 32.5 - sorted_vals = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0] - assert _percentile_linear(sorted_vals, 0.25) == 32.5 - assert _percentile_linear(sorted_vals, 0.75) == 77.5 - # n=1 → return the single value. - assert _percentile_linear([42.0], 0.25) == 42.0 - # n=0 → 0.0 (defensive). - assert _percentile_linear([], 0.25) == 0.0 - - # Bug #2 — zero-padded string IDs misclassified as sequential. - def test_bug2_zero_padded_strings_not_sequential(self) -> None: - from headroom.transforms.smart_crusher import _detect_sequential_pattern - - # Pre-fix: int("001") → 1, so ["001",...,"005"] looked like 1..5 - # and was classified as sequential. Post-fix: had_non_string_numeric - # stays False because every value came from a string → return False. - zero_padded = ["001", "002", "003", "004", "005"] - assert _detect_sequential_pattern(zero_padded, check_order=False) is False - - # Real ints (not strings) still classify as sequential. - real_ints = [1, 2, 3, 4, 5] - assert _detect_sequential_pattern(real_ints, check_order=False) is True - - # Bug #3 — rare-status detection cardinality cap. - def test_bug3_high_cardinality_pareto(self) -> None: - from headroom.transforms.smart_crusher import _detect_rare_status_values - - # 60×INFO + 25×WARN + 15 distinct error codes (cardinality 17). - # Pre-fix: 17 > 10 → field skipped → 0 outliers. - # Post-fix: top-2 covers 85% (60+25=85), K=2 ≤ 5, the 15 rare codes flagged. - items = [] - for _ in range(60): - items.append({"code": "INFO"}) - for _ in range(25): - items.append({"code": "WARN"}) - for i in range(15): - items.append({"code": f"ERR_{i}"}) - outliers = _detect_rare_status_values(items, common_fields={"code"}) - assert len(outliers) == 15 - - def test_bug3_uniform_distribution_no_outliers(self) -> None: - # 50 distinct values, 1 each → top-K never reaches 80% with K<=5. - # Field correctly identified as non-categorical. - from headroom.transforms.smart_crusher import _detect_rare_status_values - - items = [{"code": f"CAT_{i}"} for i in range(50)] - assert _detect_rare_status_values(items, common_fields={"code"}) == [] - - # Bug #4 — k-split overshoot when k_total=1. - def test_bug4_k_split_no_overshoot_when_k_total_one(self) -> None: - from headroom import SmartCrusherConfig - from headroom.transforms.smart_crusher import SmartCrusher - - config = SmartCrusherConfig(min_items_to_analyze=1) - crusher = SmartCrusher(config=config) - # Force k_total=1 by passing a single-item list — the n<=8 fast - # path returns n=1, so k_total=1. - k_total, k_first, k_last, k_importance = crusher._compute_k_split(["only"], 1.0) - assert k_total == 1 - assert k_first + k_last <= k_total, ( - f"BUG #4: k_first={k_first} + k_last={k_last} must not exceed k_total={k_total}" - ) +# Stage 3c.1 lockstep bug-fix tests previously lived here; they probed +# Python helpers (`_percentile_linear`, `_detect_sequential_pattern`, +# `_detect_rare_status_values`, `_compute_k_split`) that were removed +# along with the Python implementation in Stage 3c.1b. The Rust port +# pins the same invariants — see the `bug1_*` / `bug2_*` / `bug3_*` / +# `bug4_*` tests in `crates/headroom-core/src/transforms/smart_crusher/` +# (notably `crushers.rs` and `analyzer.rs`). Parity fixtures +# (`tests/parity/fixtures/smart_crusher/`) byte-compare the post-fix +# behavior across the language boundary. diff --git a/tests/test_transforms/test_universal_json_crush.py b/tests/test_transforms/test_universal_json_crush.py deleted file mode 100644 index 4d6eb16fa..000000000 --- a/tests/test_transforms/test_universal_json_crush.py +++ /dev/null @@ -1,489 +0,0 @@ -"""Tests for universal JSON compression (all array types). - -Verifies that SmartCrusher handles arrays of dicts, strings, numbers, -mixed types, and nested arrays — with consistent safety guarantees -across all types. -""" - -from __future__ import annotations - -import json - -import pytest - -from headroom.transforms.smart_crusher import ( - ArrayType, - SmartCrusher, - SmartCrusherConfig, - _classify_array, -) - -# ===================================================================== -# Fixtures -# ===================================================================== - - -@pytest.fixture -def crusher(): - """SmartCrusher configured for testing.""" - return SmartCrusher( - config=SmartCrusherConfig( - min_items_to_analyze=5, - min_tokens_to_crush=0, # Always crush - max_items_after_crush=15, - ) - ) - - -@pytest.fixture -def crusher_large_k(): - """SmartCrusher with higher max items for larger test arrays.""" - return SmartCrusher( - config=SmartCrusherConfig( - min_items_to_analyze=5, - min_tokens_to_crush=0, - max_items_after_crush=50, - ) - ) - - -# ===================================================================== -# Type Classification -# ===================================================================== - - -class TestClassifyArray: - def test_dict_array(self): - assert _classify_array([{"a": 1}, {"b": 2}]) == ArrayType.DICT_ARRAY - - def test_string_array(self): - assert _classify_array(["hello", "world", "foo"]) == ArrayType.STRING_ARRAY - - def test_number_array_int(self): - assert _classify_array([1, 2, 3]) == ArrayType.NUMBER_ARRAY - - def test_number_array_float(self): - assert _classify_array([1.0, 2.5, 3.7]) == ArrayType.NUMBER_ARRAY - - def test_number_array_mixed_int_float(self): - assert _classify_array([1, 2.5, 3]) == ArrayType.NUMBER_ARRAY - - def test_bool_array(self): - assert _classify_array([True, False, True]) == ArrayType.BOOL_ARRAY - - def test_nested_array(self): - assert _classify_array([[1, 2], [3, 4]]) == ArrayType.NESTED_ARRAY - - def test_mixed_array(self): - assert _classify_array([{"a": 1}, "string", 42]) == ArrayType.MIXED_ARRAY - - def test_empty(self): - assert _classify_array([]) == ArrayType.EMPTY - - def test_single_dict(self): - assert _classify_array([{"key": "val"}]) == ArrayType.DICT_ARRAY - - def test_single_string(self): - assert _classify_array(["only"]) == ArrayType.STRING_ARRAY - - def test_none_values(self): - # Arrays with None mixed in are MIXED - assert _classify_array([1, None, 3]) == ArrayType.MIXED_ARRAY - - def test_bool_not_confused_with_int(self): - # Python's True/False are int subclasses — we handle this - assert _classify_array([True, False]) == ArrayType.BOOL_ARRAY - # But mixed bools and ints should be MIXED or NUMBER depending on impl - result = _classify_array([True, 42]) - assert result in (ArrayType.MIXED_ARRAY, ArrayType.NUMBER_ARRAY) - - -# ===================================================================== -# String Array Compression -# ===================================================================== - - -class TestCrushStringArray: - def test_basic_compression(self, crusher): - strings = [f"item_{i}" for i in range(100)] - crushed, strategy = crusher._crush_string_array(strings) - assert len(crushed) < len(strings) - assert "string:adaptive" in strategy - - def test_errors_always_preserved(self, crusher): - strings = ["ok"] * 50 + ["error: connection timeout", "failed: auth denied"] + ["ok"] * 48 - crushed, strategy = crusher._crush_string_array(strings) - assert any("error" in s for s in crushed) - assert any("failed" in s for s in crushed) - - def test_first_last_kept(self, crusher): - strings = [f"item_{i}" for i in range(50)] - crushed, strategy = crusher._crush_string_array(strings) - # First item always present - assert strings[0] in crushed - # Last item always present - assert strings[-1] in crushed - - def test_dedup_reduces_output(self, crusher): - # 95 identical + 5 unique - strings = ["repeated_value"] * 95 + [f"unique_{i}" for i in range(5)] - crushed, strategy = crusher._crush_string_array(strings) - # Should massively reduce — not keep 95 copies - assert len(crushed) < 20 - # All 5 unique values should survive (they have high info value) - for i in range(5): - assert f"unique_{i}" in crushed - - def test_below_threshold_passthrough(self, crusher): - strings = ["a", "b", "c"] # Below min_items_to_analyze=5 - # Direct method call — should passthrough since <= 8 - crushed, strategy = crusher._crush_string_array(strings) - assert crushed == strings - assert "passthrough" in strategy - - def test_empty_strings_handled(self, crusher): - strings = [""] * 20 - crushed, strategy = crusher._crush_string_array(strings) - # Should not crash - assert isinstance(crushed, list) - - def test_unicode_strings(self, crusher): - strings = [f"日本語テスト_{i}" for i in range(50)] - crushed, strategy = crusher._crush_string_array(strings) - assert len(crushed) < len(strings) - assert all(isinstance(s, str) for s in crushed) - - def test_length_anomalies_preserved(self, crusher_large_k): - # Most strings are short, one is very long - strings = ["short"] * 95 + ["x" * 10000] + ["short"] * 4 - crushed, strategy = crusher_large_k._crush_string_array(strings) - assert any(len(s) > 1000 for s in crushed) - - -# ===================================================================== -# Number Array Compression -# ===================================================================== - - -class TestCrushNumberArray: - def test_basic_compression(self, crusher): - numbers = [42.0 + i * 0.1 for i in range(100)] - crushed, strategy = crusher._crush_number_array(numbers) - assert len(crushed) < len(numbers) - assert "number:adaptive" in strategy - - def test_stats_in_strategy_not_array(self, crusher): - numbers = list(range(100)) - crushed, strategy = crusher._crush_number_array(numbers) - # Stats should be in the strategy string, not in the array - assert "min=" in strategy - assert "max=" in strategy - # Array should contain only numbers (schema-preserving) - for item in crushed: - assert isinstance(item, int | float) - - def test_outliers_preserved(self, crusher): - # Normal values around 50 with one extreme outlier - numbers = [50.0 + i * 0.01 for i in range(100)] + [999.9] - crushed, strategy = crusher._crush_number_array(numbers) - assert 999.9 in crushed - assert "outliers" in strategy - - def test_all_identical(self, crusher): - numbers = [42.0] * 100 - crushed, strategy = crusher._crush_number_array(numbers) - # With all identical, should compress heavily - # Summary + a few representatives - numeric_values = [v for v in crushed if isinstance(v, int | float)] - assert all(v == 42.0 for v in numeric_values) - - def test_first_last_kept(self, crusher): - numbers = list(range(50)) - crushed, strategy = crusher._crush_number_array(numbers) - numeric_values = [v for v in crushed if isinstance(v, int | float)] - assert 0 in numeric_values # First - assert 49 in numeric_values # Last - - def test_change_point_preserved(self, crusher_large_k): - # Stable at 10, then jumps to 100 - numbers = [10.0] * 50 + [100.0] * 50 - crushed, strategy = crusher_large_k._crush_number_array(numbers) - numeric_values = [v for v in crushed if isinstance(v, int | float)] - # Both 10.0 and 100.0 should be present - assert 10.0 in numeric_values - assert 100.0 in numeric_values - - def test_nan_inf_filtered(self, crusher): - numbers = [1.0, 2.0, float("nan"), float("inf"), 3.0] * 10 - crushed, strategy = crusher._crush_number_array(numbers) - # Should not crash; stats in strategy based on finite values - assert "min=" in strategy - assert "max=" in strategy - - def test_integers_preserved_as_int(self, crusher): - numbers = list(range(50)) - crushed, strategy = crusher._crush_number_array(numbers) - numeric_values = [v for v in crushed if isinstance(v, int | float)] - # Integers should remain integers (not converted to float) - assert any(isinstance(v, int) for v in numeric_values) - - def test_statistics_accuracy(self, crusher): - numbers = list(range(1, 101)) # 1 to 100 - crushed, strategy = crusher._crush_number_array(numbers) - # Stats are in the strategy string - assert "min=1" in strategy - assert "max=100" in strategy - assert "mean=50.5" in strategy - - -# ===================================================================== -# Mixed Array Compression -# ===================================================================== - - -class TestCrushMixedArray: - def test_basic_compression(self, crusher_large_k): - mixed = [{"id": i} for i in range(30)] + [f"msg_{i}" for i in range(30)] - crushed, strategy = crusher_large_k._crush_mixed_array(mixed) - # With diversity-aware K, unique items may all be kept. - # Verify compression happened OR items are preserved due to high diversity. - assert len(crushed) <= len(mixed) - assert "mixed" in strategy - - def test_small_groups_kept(self, crusher): - # 50 dicts + 3 strings (below threshold) - mixed = [{"id": i} for i in range(50)] + ["rare_1", "rare_2", "rare_3"] - crushed, strategy = crusher._crush_mixed_array(mixed) - # All 3 rare strings should be kept (below min_items threshold) - assert "rare_1" in crushed - assert "rare_2" in crushed - assert "rare_3" in crushed - - def test_errors_across_types(self, crusher_large_k): - mixed = ( - [{"status": "ok"}] * 30 - + [{"status": "error: timeout"}] - + ["error: auth failed"] - + [f"ok_{i}" for i in range(30)] - ) - crushed, strategy = crusher_large_k._crush_mixed_array(mixed) - crushed_str = json.dumps(crushed) - assert "error: timeout" in crushed_str - assert "error: auth failed" in crushed_str - - def test_original_order_preserved(self, crusher_large_k): - mixed = [{"id": i} for i in range(20)] + [f"str_{i}" for i in range(20)] - crushed, strategy = crusher_large_k._crush_mixed_array(mixed) - # Dicts should come before strings (original order) - first_str_idx = next( - (i for i, item in enumerate(crushed) if isinstance(item, str)), len(crushed) - ) - last_dict_idx = max( - (i for i, item in enumerate(crushed) if isinstance(item, dict)), default=-1 - ) - assert last_dict_idx < first_str_idx - - def test_passthrough_small(self, crusher): - mixed = [1, "two", {"three": 3}] - crushed, strategy = crusher._crush_mixed_array(mixed) - assert crushed == mixed - assert "passthrough" in strategy - - -# ===================================================================== -# Adaptive K -# ===================================================================== - - -class TestAdaptiveK: - def test_scales_with_n(self, crusher): - """K grows sublinearly with collection size (or saturates at max_items).""" - small = [f"item_{i}" for i in range(20)] - large = [f"item_{i}" for i in range(500)] - - k_small = crusher._compute_k_split(small)[0] - k_large = crusher._compute_k_split(large)[0] - - # With diversity-aware K, both may saturate at max_items_after_crush - # for highly unique items. The key property: k never exceeds max_items. - assert k_large >= k_small - assert k_large <= crusher.config.max_items_after_crush - - def test_respects_max_items(self, crusher): - items = [f"item_{i}" for i in range(1000)] - k_total, _, _, _ = crusher._compute_k_split(items) - assert k_total <= crusher.config.max_items_after_crush - - def test_first_last_fractions(self, crusher): - items = [f"item_{i}" for i in range(100)] - k_total, k_first, k_last, k_importance = crusher._compute_k_split(items) - # First and last should be roughly the configured fractions - assert k_first >= 1 - assert k_last >= 1 - assert k_first + k_last + k_importance == k_total - - def test_homogeneous_vs_diverse(self, crusher_large_k): - """Homogeneous data should produce smaller K than diverse data.""" - homogeneous = ["same value"] * 100 - diverse = [f"unique_value_{i}_{'x' * (i * 10)}" for i in range(100)] - - k_homo = crusher_large_k._compute_k_split(homogeneous)[0] - k_diverse = crusher_large_k._compute_k_split(diverse)[0] - - # Diverse should need more items (or at least equal) - assert k_diverse >= k_homo - - -# ===================================================================== -# Safety Guarantees (parametrized across types) -# ===================================================================== - - -class TestSafetyGuarantees: - @pytest.mark.parametrize( - "items,item_type", - [ - ([{"status": "ok"}] * 50 + [{"status": "error: timeout"}], "dict"), - (["ok"] * 50 + ["error: connection failed"], "string"), - ( - [{"id": i} for i in range(30)] - + ["error: auth denied"] - + [f"ok_{i}" for i in range(30)], - "mixed", - ), - ], - ids=["dict_array", "string_array", "mixed_array"], - ) - def test_errors_never_dropped(self, crusher_large_k, items, item_type): - """Error items must be preserved regardless of array type.""" - if item_type == "dict": - crushed, _ = crusher_large_k._crush_string_array([json.dumps(i) for i in items]) - crushed_text = " ".join(crushed) - elif item_type == "string": - crushed, _ = crusher_large_k._crush_string_array(items) - crushed_text = " ".join(crushed) - elif item_type == "mixed": - crushed, _ = crusher_large_k._crush_mixed_array(items) - crushed_text = json.dumps(crushed) - - assert "error" in crushed_text.lower() - - @pytest.mark.parametrize( - "items", - [ - [f"item_{i}" for i in range(50)], - list(range(50)), - ], - ids=["string_array", "number_array"], - ) - def test_first_last_always_present(self, crusher, items): - """First and last items must be present in output.""" - if isinstance(items[0], str): - crushed, _ = crusher._crush_string_array(items) - assert items[0] in crushed - assert items[-1] in crushed - else: - crushed, _ = crusher._crush_number_array(items) - numeric = [v for v in crushed if isinstance(v, int | float)] - assert items[0] in numeric - assert items[-1] in numeric - - @pytest.mark.parametrize( - "items", - [ - ["a", "b", "c"], - [1, 2, 3], - [{"x": 1}, "y", 2], - ], - ids=["string_below", "number_below", "mixed_below"], - ) - def test_passthrough_below_min_items(self, crusher, items): - """Arrays below min_items_to_analyze pass through unchanged.""" - if all(isinstance(i, str) for i in items): - crushed, strategy = crusher._crush_string_array(items) - elif all(isinstance(i, int | float) for i in items): - crushed, strategy = crusher._crush_number_array(items) - else: - crushed, strategy = crusher._crush_mixed_array(items) - assert "passthrough" in strategy - - -# ===================================================================== -# Integration: Full pipeline -# ===================================================================== - - -class TestFullPipelineIntegration: - """Test that new types work through the compress() API.""" - - def test_string_array_via_compress(self): - from headroom import compress - - strings = [f"log line {i}: GET /api 200" for i in range(100)] - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "user", "content": "Show logs"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "get_logs", "arguments": "{}"}, - } - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(strings)}, - ] - result = compress(messages) - assert result.tokens_saved > 0 - assert result.compression_ratio > 0 - - def test_number_array_via_compress(self): - from headroom import compress - - numbers = [42.0 + i * 0.1 for i in range(200)] - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "user", "content": "Show metrics"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "get_metrics", "arguments": "{}"}, - } - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(numbers)}, - ] - result = compress(messages) - assert result.tokens_saved > 0 - - def test_dict_array_unchanged(self): - """Verify dict arrays still work (regression test).""" - from headroom import compress - - data = [{"id": i, "name": f"user_{i}", "status": "active"} for i in range(100)] - messages = [ - {"role": "system", "content": "You are helpful."}, - {"role": "user", "content": "List users"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "list_users", "arguments": "{}"}, - } - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(data)}, - ] - result = compress(messages) - assert result.tokens_saved > 0 - assert result.compression_ratio > 0