diff --git a/.github/workflows/devcontainers.yml b/.github/workflows/devcontainers.yml index ea7bce257..04552b676 100644 --- a/.github/workflows/devcontainers.yml +++ b/.github/workflows/devcontainers.yml @@ -85,6 +85,22 @@ jobs: steps: - uses: actions/checkout@v6 + # The worktree devcontainer runs the same `uv sync --extra dev` build as + # the default validate job, which now pulls transformers 5.x. Copying that + # into the venv volume exhausts the GitHub runner's disk ("No space left on + # device", see PR #495). Reclaim ~14 GB by stripping preinstalled tools the + # build never touches — mirrors the memory-stack job's existing remedy. + - name: Free runner disk + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false + - name: Create linked worktree run: git worktree add "$RUNNER_TEMP/headroom-worktree" HEAD diff --git a/examples/README.md b/examples/README.md index 117a5849c..6251e7092 100644 --- a/examples/README.md +++ b/examples/README.md @@ -31,6 +31,17 @@ export OPENAI_API_KEY='your-key' python examples/streaming_example.py ``` +### tabular_compression_demo.py + +Tabular + spreadsheet compression on generated sample data (no API key needed). +Shows where CSV/markdown tables and `.xlsx` workbooks compress and where compact, +all-unique data correctly passes through: + +```bash +python examples/tabular_compression_demo.py # run all scenarios +python examples/tabular_compression_demo.py --write DIR # also save the sample files +``` + ## Evaluation Examples ### smart_vs_naive_eval.py diff --git a/examples/tabular_compression_demo.py b/examples/tabular_compression_demo.py new file mode 100644 index 000000000..78994ce53 --- /dev/null +++ b/examples/tabular_compression_demo.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Demo / test harness for tabular + spreadsheet compression. + +Generates representative sample data and runs it through Headroom's tabular +compressor so you can see where it helps (verbose / redundant tables, and +query-driven selection) and where it correctly does nothing (compact, all-unique +data with no signal to compress against). + +Usage: + python examples/tabular_compression_demo.py # run all scenarios + python examples/tabular_compression_demo.py --write DIR # also save sample files + +The .xlsx scenario requires the spreadsheet extra: + pip install headroom-ai[spreadsheet] +""" + +from __future__ import annotations + +import argparse +import importlib.util +from pathlib import Path + +import headroom +from headroom.transforms.content_router import ContentRouter + +_HAS_OPENPYXL = importlib.util.find_spec("openpyxl") is not None + + +# ─── Sample data generators ───────────────────────────────────────────────── + + +def compact_unique_csv(rows: int = 60) -> str: + """Minimal CSV, every row unique — nothing safely removable (~0 savings).""" + lines = ["id,name,age,city"] + lines += [f"{i},user_{i},{20 + i % 50},city_{i}" for i in range(rows)] + return "\n".join(lines) + + +def redundant_csv(rows: int = 120) -> str: + """Highly repetitive rows — SmartCrusher can dedupe (big savings).""" + lines = ["region,product,status"] + lines += ["EMEA,widget-A,shipped" for _ in range(rows)] + return "\n".join(lines) + + +def verbose_markdown(rows: int = 40) -> str: + """A padded markdown table — verbose source, lossless compaction wins.""" + header = "| name | age | city | status | dept |\n| --- | --- | --- | --- | --- |" + body = "\n".join( + f"| user_{i} | {20 + i} | city_{i % 5} | active | engineering |" for i in range(rows) + ) + return f"{header}\n{body}" + + +# ─── Runners ──────────────────────────────────────────────────────────────── + + +def _run_router(label: str, content: str) -> None: + """Compress raw tabular text through the ContentRouter.""" + result = ContentRouter().compress(content) + before = len(content) + after = len(result.compressed) + pct = 100 * (before - after) / before if before else 0.0 + print( + f"{label:24s} strat={result.strategy_used.value:9s} " + f"chars {before:6d} -> {after:6d} ({pct:5.1f}% saved)" + ) + + +def _run_messages(label: str, content: str) -> None: + """Compress via the full pipeline (real tokenizer accounting).""" + res = headroom.compress( + [{"role": "user", "content": content}], + compress_user_messages=True, + ) + pct = 100 * res.tokens_saved / res.tokens_before if res.tokens_before else 0.0 + print( + f"{label:24s} tokens {res.tokens_before:6d} -> " + f"{res.tokens_after:6d} ({pct:5.1f}% saved)" + ) + + +def _run_xlsx(label: str, path: Path) -> None: + res = headroom.compress_spreadsheet(str(path)) + pct = 100 * res.tokens_saved / res.tokens_before if res.tokens_before else 0.0 + print( + f"{label:24s} tokens {res.tokens_before:6d} -> " + f"{res.tokens_after:6d} ({pct:5.1f}% saved)" + ) + + +def _build_xlsx(path: Path) -> None: + import openpyxl + + wb = openpyxl.Workbook() + unique = wb.active + unique.title = "Unique" + unique.append(["id", "name", "dept"]) + for i in range(60): + unique.append([i, f"user_{i}", ["eng", "sales", "ops"][i % 3]]) + + redundant = wb.create_sheet("Redundant") + redundant.append(["region", "product", "status"]) + for _ in range(120): + redundant.append(["EMEA", "widget-A", "shipped"]) + + wb.save(path) + + +# ─── Main ─────────────────────────────────────────────────────────────────── + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--write", + metavar="DIR", + help="Also write the generated sample files (.csv/.md/.xlsx) to DIR", + ) + args = parser.parse_args() + + samples = { + "compact_unique.csv": compact_unique_csv(), + "redundant.csv": redundant_csv(), + "verbose_table.md": verbose_markdown(), + } + + print("=== Raw tabular text (ContentRouter, char-level) ===") + _run_router("compact unique CSV", samples["compact_unique.csv"]) + _run_router("redundant CSV", samples["redundant.csv"]) + _run_router("verbose markdown", samples["verbose_table.md"]) + + print("\n=== Full pipeline (real tokenizer) ===") + _run_messages("redundant CSV", samples["redundant.csv"]) + + print("\n=== Binary spreadsheet (.xlsx) ===") + if not _HAS_OPENPYXL: + print(" skipped — install: pip install headroom-ai[spreadsheet]") + else: + out_dir = Path(args.write) if args.write else Path("/tmp") + out_dir.mkdir(parents=True, exist_ok=True) + xlsx_path = out_dir / "demo.xlsx" + _build_xlsx(xlsx_path) + _run_xlsx("2-sheet workbook", xlsx_path) + + if args.write: + out = Path(args.write) + out.mkdir(parents=True, exist_ok=True) + for name, content in samples.items(): + (out / name).write_text(content) + print(f"\nSample files written to {out.resolve()}") + + print( + "\nTakeaway: redundant/verbose tables compress; compact all-unique data " + "correctly passes through (lossless-only — nothing safely removable)." + ) + + +if __name__ == "__main__": + main() diff --git a/headroom/__init__.py b/headroom/__init__.py index b168df9eb..3f73656ce 100644 --- a/headroom/__init__.py +++ b/headroom/__init__.py @@ -74,7 +74,7 @@ from importlib import import_module from typing import Any from ._version import __version__ # noqa: F401 -from .compress import CompressConfig, CompressResult, compress +from .compress import CompressConfig, CompressResult, compress, compress_spreadsheet # Keep a real callable bound for the one-function compression API so # `from headroom import compress` is never shadowed by the submodule object. @@ -165,6 +165,7 @@ __all__ = [ "EmbedderBackend", # One-function compression API "compress", + "compress_spreadsheet", "CompressConfig", "CompressResult", # Hooks @@ -261,6 +262,7 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = { "reset_otel_metrics": ("headroom.observability", "reset_otel_metrics"), # One-function API "compress": ("headroom.compress", "compress"), + "compress_spreadsheet": ("headroom.compress", "compress_spreadsheet"), # Hooks "CompressionHooks": ("headroom.hooks", "CompressionHooks"), "CompressContext": ("headroom.hooks", "CompressContext"), diff --git a/headroom/compress.py b/headroom/compress.py index 47ab8898e..0b967026b 100644 --- a/headroom/compress.py +++ b/headroom/compress.py @@ -349,6 +349,39 @@ def compress( ) +def compress_spreadsheet( + path: str, + model: str = "claude-sonnet-4-5-20250929", + model_limit: int = 200000, + **kwargs: Any, +) -> CompressResult: + """Compress a binary spreadsheet (``.xlsx`` / ``.xls``). + + Each sheet is rendered to CSV text and submitted as its own user message so + the tabular compressor (CSV → SmartCrusher, lossless-first + lossy CCR + fallback) is applied per sheet. Requires the ``spreadsheet`` extra + (``pip install headroom-ai[spreadsheet]``). + + Args: + path: Path to a ``.xlsx`` or ``.xls`` file. + model: Model name (token counting / context limit). + model_limit: Model context window size in tokens. + **kwargs: Forwarded to :func:`compress` (e.g. ``target_ratio``). + + Returns: + CompressResult over the per-sheet messages. + """ + from headroom.transforms.spreadsheet_ingest import load_spreadsheet + + sheets = load_spreadsheet(path) + messages = [{"role": "user", "content": text} for text in sheets.values()] + if not messages: + return CompressResult(messages=[]) + # User messages hold the table text, so they must be compressible here. + kwargs.setdefault("compress_user_messages", True) + return compress(messages, model=model, model_limit=model_limit, **kwargs) + + def _get_pipeline() -> Any: """Get or create the singleton compression pipeline.""" global _pipeline diff --git a/headroom/transforms/__init__.py b/headroom/transforms/__init__.py index 1f64182f8..47932bc5c 100644 --- a/headroom/transforms/__init__.py +++ b/headroom/transforms/__init__.py @@ -61,6 +61,11 @@ if TYPE_CHECKING: SearchCompressorConfig, ) from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig # noqa: F401 + from headroom.transforms.tabular_ingest import ( # noqa: F401 + TabularCompressionResult, + TabularCompressor, + TabularCompressorConfig, + ) _HTML_EXTRACTOR_AVAILABLE = importlib.util.find_spec("trafilatura") is not None @@ -88,6 +93,9 @@ __all__ = [ "LogCompressor", "LogCompressorConfig", "LogCompressionResult", + "TabularCompressor", + "TabularCompressorConfig", + "TabularCompressionResult", "DiffCompressor", "DiffCompressorConfig", "DiffCompressionResult", @@ -154,6 +162,15 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = { "LogCompressor": ("headroom.transforms.log_compressor", "LogCompressor"), "LogCompressorConfig": ("headroom.transforms.log_compressor", "LogCompressorConfig"), "LogCompressionResult": ("headroom.transforms.log_compressor", "LogCompressionResult"), + "TabularCompressor": ("headroom.transforms.tabular_ingest", "TabularCompressor"), + "TabularCompressorConfig": ( + "headroom.transforms.tabular_ingest", + "TabularCompressorConfig", + ), + "TabularCompressionResult": ( + "headroom.transforms.tabular_ingest", + "TabularCompressionResult", + ), "DiffCompressor": ("headroom.transforms.diff_compressor", "DiffCompressor"), "DiffCompressorConfig": ("headroom.transforms.diff_compressor", "DiffCompressorConfig"), "DiffCompressionResult": ( diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.py index 324139726..78ab52fb5 100644 --- a/headroom/transforms/content_detector.py +++ b/headroom/transforms/content_detector.py @@ -30,6 +30,7 @@ class ContentType(Enum): BUILD_OUTPUT = "build" # Compiler, test, lint logs GIT_DIFF = "diff" # Unified diff format HTML = "html" # Web pages (needs content extraction, not compression) + TABULAR = "tabular" # CSV/TSV, markdown tables, fixed-width tables PLAIN_TEXT = "text" # Fallback @@ -47,6 +48,10 @@ _SEARCH_RESULT_PATTERN = re.compile( r"^[^\s:]+:\d+:" # file:line: format (grep -n style) ) +# A markdown table separator row, e.g. "| --- | :--: |" or "---|---". +# Every cell must be dashes with optional alignment colons. +_MD_SEP_CELL = re.compile(r"^:?-{2,}:?$") + # Bug-fix (2026-04-25): extended to recognize merge-commit headers # (`diff --combined `, `diff --cc `) and combined-diff hunk # headers (`@@@`+ ranges). Previously only `git diff` shape was detected, @@ -159,12 +164,20 @@ def detect_content_type(content: str) -> DetectionResult: if log_result and log_result.confidence >= 0.5: return log_result - # 6. Check for source code + # 6. Check for tabular data (CSV/TSV, markdown tables). Runs after + # search/log so colon-delimited search output and freeform logs claim + # their content first; tabular requires a consistent multi-column + # delimiter or a markdown header+separator pair. + tabular_result = _try_detect_tabular(content) + if tabular_result and tabular_result.confidence >= 0.6: + return tabular_result + + # 7. Check for source code code_result = _try_detect_code(content) if code_result and code_result.confidence >= 0.5: return code_result - # 7. Fallback to plain text + # 8. Fallback to plain text return DetectionResult(ContentType.PLAIN_TEXT, 0.5, {}) @@ -380,6 +393,102 @@ def _try_detect_log(content: str) -> DetectionResult | None: ) +def _md_cell_count(row: str) -> int: + """Count cells in a markdown table row, ignoring the outer pipes.""" + return len(row.strip().strip("|").split("|")) + + +def _is_md_separator(row: str) -> bool: + """True if `row` is a markdown table separator (e.g. ``| --- | :--: |``).""" + cells = [c.strip() for c in row.strip().strip("|").split("|")] + cells = [c for c in cells if c != ""] + if len(cells) < 2: + return False + return all(_MD_SEP_CELL.match(c) for c in cells) + + +def _try_detect_markdown_table(lines: list[str]) -> DetectionResult | None: + """Detect a markdown table: a piped header row followed by a separator.""" + for i in range(len(lines) - 1): + header, sep = lines[i], lines[i + 1] + if "|" in header and _is_md_separator(sep): + cols = _md_cell_count(header) + if cols >= 2: + return DetectionResult( + ContentType.TABULAR, + 0.95, + {"format": "markdown", "columns": cols}, + ) + return None + + +def _try_detect_delimited(lines: list[str]) -> DetectionResult | None: + """Detect CSV/TSV by a delimiter with a consistent per-line column count. + + A stable column count is what separates real tabular data from prose that + merely contains commas, and from ``file:line:content`` search output (which + has a variable number of colons). Tabs are a stronger signal than commas + (they rarely occur in prose), so they need less consistency. + """ + from collections import Counter + + sample = lines[:20] + if len(sample) < 3: + return None + + best: DetectionResult | None = None + for delim, min_consistency in ((",", 0.85), ("\t", 0.7), (";", 0.85), ("|", 0.85)): + counts = [row.count(delim) for row in sample] + if counts[0] == 0: # header row must contain the delimiter + continue + common_count, freq = Counter(counts).most_common(1)[0] + if common_count == 0: + continue + consistency = freq / len(sample) + ncols = common_count + 1 + if ncols < 2 or consistency < min_consistency: + continue + # Prose guard: prose that merely contains commas ("Hello, friend.") + # reads like sentences. Real table rows are short field tuples. + if _looks_like_prose(sample, delim): + continue + confidence = min(0.95, 0.5 + consistency * 0.3 + min(ncols, 5) * 0.03) + if best is None or confidence > best.confidence: + best = DetectionResult( + ContentType.TABULAR, + confidence, + {"format": "csv", "delimiter": delim, "columns": ncols}, + ) + return best + + +def _looks_like_prose(sample: list[str], delim: str) -> bool: + """Heuristic: distinguish comma-bearing prose from real CSV rows. + + Prose reads like sentences (ends with ``.!?``) and has wordy cells; CSV + rows are short field tuples. Either signal rejects the candidate. + """ + enders = sum(1 for r in sample if r.rstrip().endswith((".", "!", "?"))) + if enders / len(sample) >= 0.5: + return True + cells = [c.strip() for r in sample for c in r.split(delim)] + avg_words = sum(len(c.split()) for c in cells) / len(cells) + return avg_words > 3 + + +def _try_detect_tabular(content: str) -> DetectionResult | None: + """Detect tabular text: markdown tables first, then delimited CSV/TSV.""" + lines = [ln for ln in content.split("\n") if ln.strip()][:50] + if len(lines) < 3: + return None + + md_result = _try_detect_markdown_table(lines) + if md_result: + return md_result + + return _try_detect_delimited(lines) + + def _try_detect_code(content: str) -> DetectionResult | None: """Try to detect source code and identify language.""" lines = content.split("\n")[:100] # Check first 100 lines diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py index f8082b06a..901878eeb 100644 --- a/headroom/transforms/content_router.py +++ b/headroom/transforms/content_router.py @@ -457,6 +457,7 @@ class CompressionStrategy(Enum): TEXT = "text" DIFF = "diff" HTML = "html" + TABULAR = "tabular" MIXED = "mixed" PASSTHROUGH = "passthrough" @@ -577,6 +578,7 @@ class ContentRouterConfig: enable_smart_crusher: Enable JSON array compression. enable_search_compressor: Enable search result compression. enable_log_compressor: Enable build/test log compression. + enable_tabular_compressor: Enable CSV/TSV/markdown-table compression. enable_image_optimizer: Enable image token optimization. prefer_code_aware_for_code: Use CodeAware over Kompress for code. mixed_content_threshold: Min distinct types to consider "mixed". @@ -593,6 +595,7 @@ class ContentRouterConfig: enable_smart_crusher: bool = True enable_search_compressor: bool = True enable_log_compressor: bool = True + enable_tabular_compressor: bool = True # CSV/TSV/markdown tables via SmartCrusher enable_html_extractor: bool = True # HTML content extraction enable_image_optimizer: bool = True # Image token optimization @@ -935,6 +938,7 @@ class ContentRouter(Transform): self._log_compressor: Any = None self._diff_compressor: Any = None self._html_extractor: Any = None + self._tabular_compressor: Any = None self._kompress: Any = None # TOIN integration for cross-strategy learning @@ -1232,6 +1236,7 @@ class ContentRouter(Transform): ContentType.BUILD_OUTPUT: CompressionStrategy.LOG, ContentType.GIT_DIFF: CompressionStrategy.DIFF, ContentType.HTML: CompressionStrategy.HTML, + ContentType.TABULAR: CompressionStrategy.TABULAR, ContentType.PLAIN_TEXT: CompressionStrategy.TEXT, } @@ -1466,6 +1471,18 @@ class ContentRouter(Transform): ) decision_reason = "log_compressor" + elif strategy == CompressionStrategy.TABULAR: + if self.config.enable_tabular_compressor: + compressor = self._get_tabular_compressor() + if compressor: + compressor_name = type(compressor).__name__ + result = compressor.compress(content, context=context, bias=bias) + compressed, compressed_tokens = ( + result.compressed, + len(result.compressed.split()), + ) + decision_reason = "tabular_compressor" + elif strategy == CompressionStrategy.DIFF: compressor = self._get_diff_compressor() if compressor: @@ -1516,6 +1533,7 @@ class ContentRouter(Transform): fallback_eligible_strategy = strategy in { CompressionStrategy.SMART_CRUSHER, CompressionStrategy.CODE_AWARE, + CompressionStrategy.TABULAR, } fallback_no_savings = compressed == content or compressed_tokens >= original_tokens if fallback_eligible_strategy and fallback_no_savings: @@ -1698,6 +1716,7 @@ class ContentRouter(Transform): ContentType.BUILD_OUTPUT: CompressionStrategy.LOG, ContentType.GIT_DIFF: CompressionStrategy.DIFF, ContentType.HTML: CompressionStrategy.HTML, + ContentType.TABULAR: CompressionStrategy.TABULAR, ContentType.PLAIN_TEXT: CompressionStrategy.TEXT, } return mapping.get(content_type, self.config.fallback_strategy) @@ -1711,6 +1730,7 @@ class ContentRouter(Transform): CompressionStrategy.LOG: ContentType.BUILD_OUTPUT, CompressionStrategy.DIFF: ContentType.GIT_DIFF, CompressionStrategy.HTML: ContentType.HTML, + CompressionStrategy.TABULAR: ContentType.TABULAR, CompressionStrategy.TEXT: ContentType.PLAIN_TEXT, CompressionStrategy.KOMPRESS: ContentType.PLAIN_TEXT, CompressionStrategy.PASSTHROUGH: ContentType.PLAIN_TEXT, @@ -1785,6 +1805,17 @@ class ContentRouter(Transform): logger.debug("LogCompressor not available") return self._log_compressor + def _get_tabular_compressor(self) -> Any: + """Get TabularCompressor (lazy load).""" + if self._tabular_compressor is None: + try: + from .tabular_ingest import TabularCompressor + + self._tabular_compressor = TabularCompressor() + except ImportError: # pragma: no cover - defensive; tabular_ingest is pure stdlib + logger.debug("TabularCompressor not available") + return self._tabular_compressor + def _get_diff_compressor(self) -> Any: """Get DiffCompressor (lazy load). Rust-only — Python implementation retired in Stage 3b. The wheel (`headroom._core`) is a hard import. diff --git a/headroom/transforms/spreadsheet_ingest.py b/headroom/transforms/spreadsheet_ingest.py new file mode 100644 index 000000000..01bad93e2 --- /dev/null +++ b/headroom/transforms/spreadsheet_ingest.py @@ -0,0 +1,96 @@ +"""Binary spreadsheet ingestion: ``.xlsx`` / ``.xls`` → tabular text. + +The compression pipeline is text-only, so binary spreadsheets enter through this +adapter at the SDK boundary. Each sheet is rendered to CSV text, which then flows +through the normal tabular detection → SmartCrusher path like any other table. + +Parsers are optional dependencies (``pip install headroom-ai[spreadsheet]``) and +are imported lazily; a missing dependency fails loudly with an actionable +message rather than silently degrading. +""" + +from __future__ import annotations + +import csv +import io +from pathlib import Path + +__all__ = ["load_spreadsheet"] + + +def _rows_to_csv(rows: list[list[object]]) -> str: + """Render rows to CSV text, dropping fully empty trailing rows.""" + buf = io.StringIO() + writer = csv.writer(buf) + for row in rows: + writer.writerow(["" if cell is None else cell for cell in row]) + return buf.getvalue().strip("\n") + + +def _load_xlsx(path: Path) -> dict[str, str]: + try: + import openpyxl + except ImportError as e: # pragma: no cover - openpyxl ships in [dev]; defensive guard + raise ImportError( + "Reading .xlsx files requires openpyxl. " + "Install it with: pip install headroom-ai[spreadsheet]" + ) from e + + wb = openpyxl.load_workbook(path, read_only=True, data_only=True) + sheets: dict[str, str] = {} + try: + for ws in wb.worksheets: + rows = [list(r) for r in ws.iter_rows(values_only=True)] + text = _rows_to_csv(rows) + if text.strip(): + sheets[ws.title] = text + finally: + wb.close() + return sheets + + +def _load_xls( + path: Path, +) -> dict[str, str]: # pragma: no cover - legacy .xls; needs optional xlrd + binary fixture + try: + import xlrd + except ImportError as e: + raise ImportError( + "Reading legacy .xls files requires xlrd. " + "Install it with: pip install headroom-ai[spreadsheet]" + ) from e + + book = xlrd.open_workbook(str(path)) + sheets: dict[str, str] = {} + for sheet in book.sheets(): + rows = [sheet.row_values(i) for i in range(sheet.nrows)] + text = _rows_to_csv(rows) + if text.strip(): + sheets[sheet.name] = text + return sheets + + +def load_spreadsheet(path: str | Path) -> dict[str, str]: + """Load a spreadsheet file into ``{sheet_name: csv_text}``. + + Args: + path: Path to a ``.xlsx`` or ``.xls`` file. + + Returns: + Mapping of sheet name to CSV-rendered text (empty sheets omitted). + + Raises: + FileNotFoundError: If the path does not exist. + ValueError: If the file extension is unsupported. + ImportError: If the required parser dependency is not installed. + """ + p = Path(path) + if not p.exists(): + raise FileNotFoundError(f"Spreadsheet not found: {p}") + + suffix = p.suffix.lower() + if suffix == ".xlsx": + return _load_xlsx(p) + if suffix == ".xls": + return _load_xls(p) # pragma: no cover - legacy .xls path, see _load_xls + raise ValueError(f"Unsupported spreadsheet format '{suffix}'. Supported: .xlsx, .xls") diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py new file mode 100644 index 000000000..07b889b13 --- /dev/null +++ b/headroom/transforms/tabular_ingest.py @@ -0,0 +1,218 @@ +"""Tabular-text compressor: bridges CSV/TSV/markdown tables to SmartCrusher. + +Raw tabular *text* (CSV/TSV files, markdown tables, fixed-width tables) has no +native compressor — it would otherwise fall through to plain-text Kompress, +ignoring its row/column structure. This module parses tabular text into a JSON +array of records and routes it through the existing, battle-tested +`SmartCrusher`, which already does lossless ``csv-schema`` compaction first and +lossy row-drop with reversible ``<>`` markers as a fallback. + +No new compression algorithm and no new CCR plumbing live here — only the +text→records bridge. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass + +from .content_detector import ContentType, detect_content_type + +# Mirrors content_detector's separator-cell pattern (e.g. ``| --- | :--: |``). +_MD_SEP_CELL = re.compile(r"^:?-{2,}:?$") + + +# ─── Public dataclasses (mirror SearchCompressor / LogCompressor surface) ──── + + +@dataclass +class TabularCompressorConfig: + """Configuration for tabular-text compression.""" + + # Pass-through to SmartCrusher's lossless renderer. + compaction_format: str = "csv-schema" + # Only keep SmartCrusher's output if it is strictly smaller than the + # original tabular text (already-compact CSV may not benefit losslessly). + min_savings_chars: int = 1 + + +@dataclass +class TabularCompressionResult: + """Result of tabular-text compression.""" + + compressed: str + original: str + was_modified: bool + fmt: str # "csv" | "markdown" | "fixed_width" + rows: int + columns: int + strategy: str = "tabular" + + @property + def compression_ratio(self) -> float: + if not self.original: + return 0.0 + return len(self.compressed) / len(self.original) + + +# ─── Parsers (text → headers + rows) ───────────────────────────────────────── + + +def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[str]]]: + """Parse delimited text via the stdlib csv reader.""" + reader = csv.reader(io.StringIO(content), delimiter=delimiter) + parsed = [row for row in reader if any(cell.strip() for cell in row)] + if not parsed: + return [], [] + headers = [h.strip() for h in parsed[0]] + return headers, parsed[1:] + + +def parse_markdown_table(content: str) -> tuple[list[str], list[list[str]]]: + """Parse a markdown table, dropping the ``|---|`` separator row.""" + + def split_row(row: str) -> list[str]: + return [c.strip() for c in row.strip().strip("|").split("|")] + + def is_separator(row: str) -> bool: + cells = [c for c in split_row(row) if c] + return len(cells) >= 2 and all(_MD_SEP_CELL.match(c) for c in cells) + + lines = [ln for ln in content.split("\n") if ln.strip() and "|" in ln] + if len(lines) < 2: + return [], [] + headers = split_row(lines[0]) + rows = [split_row(ln) for ln in lines[1:] if not is_separator(ln)] + return headers, rows + + +def parse_fixed_width(content: str) -> tuple[list[str], list[list[str]]]: + """Parse whitespace-aligned columns (best-effort, ≥ 2 spaces as a gap).""" + lines = [ln for ln in content.split("\n") if ln.strip()] + if len(lines) < 2: + return [], [] + splitter = re.compile(r"\s{2,}") + headers = splitter.split(lines[0].strip()) + rows = [splitter.split(ln.strip()) for ln in lines[1:]] + return headers, rows + + +def to_records(headers: list[str], rows: list[list[str]]) -> list[dict[str, str]]: + """Zip headers with each row into dicts, padding/truncating to width.""" + if not headers: + return [] + width = len(headers) + records: list[dict[str, str]] = [] + for row in rows: + padded = (row + [""] * width)[:width] + records.append({headers[i]: padded[i] for i in range(width)}) + return records + + +def parse_tabular( + content: str, +) -> tuple[list[str], list[list[str]], str] | None: + """Detect the tabular format and parse to (headers, rows, fmt). + + Returns ``None`` if the content is not tabular. + """ + detection = detect_content_type(content) + if detection.content_type is not ContentType.TABULAR: + return None + + fmt = detection.metadata.get("format", "csv") + if fmt == "markdown": + headers, rows = parse_markdown_table(content) + elif fmt == "fixed_width": + headers, rows = parse_fixed_width(content) + else: + delimiter = detection.metadata.get("delimiter", ",") + headers, rows = parse_csv(content, delimiter) + + if not headers or not rows: + return None + return headers, rows, fmt + + +# ─── Compressor (text → records → SmartCrusher) ────────────────────────────── + + +class TabularCompressor: + """Compresses tabular text by bridging it through SmartCrusher. + + Public surface mirrors the other content-type compressors so the router + and tests treat it uniformly. + """ + + def __init__(self, config: TabularCompressorConfig | None = None) -> None: + self.config = config or TabularCompressorConfig() + + def compress( + self, + content: str, + context: str = "", + bias: float = 1.0, + ) -> TabularCompressionResult: + parsed = parse_tabular(content) + if parsed is None: + return TabularCompressionResult( + compressed=content, + original=content, + was_modified=False, + fmt="unknown", + rows=0, + columns=0, + ) + + headers, rows, fmt = parsed + records = to_records(headers, rows) + json_str = json.dumps(records, ensure_ascii=False) + + # Lazy import keeps the Rust dependency off the import path until a + # tabular payload actually arrives. + from .smart_crusher import SmartCrusher + + crusher = SmartCrusher( + with_compaction=True, + compaction_format=self.config.compaction_format, + ) + result = crusher.crush(json_str, context, bias) + + # SmartCrusher compressed the JSON form; compare its output against the + # original *tabular text*. Already-compact CSV may not beat its own + # source, so only adopt the result when it genuinely saves bytes. + savings = len(content) - len(result.compressed) + if not result.was_modified or savings < self.config.min_savings_chars: + return TabularCompressionResult( + compressed=content, + original=content, + was_modified=False, + fmt=fmt, + rows=len(rows), + columns=len(headers), + ) + + return TabularCompressionResult( + compressed=result.compressed, + original=content, + was_modified=True, + fmt=fmt, + rows=len(rows), + columns=len(headers), + strategy=result.strategy or "tabular", + ) + + +__all__ = [ + "TabularCompressor", + "TabularCompressorConfig", + "TabularCompressionResult", + "parse_csv", + "parse_markdown_table", + "parse_fixed_width", + "parse_tabular", + "to_records", +] diff --git a/pyproject.toml b/pyproject.toml index 057f03014..30b520806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,6 +160,11 @@ image = [ reports = [ "jinja2>=3.0.0", ] +# Binary spreadsheet ingestion (.xlsx / .xls -> tabular text) +spreadsheet = [ + "openpyxl>=3.1.0", # .xlsx + "xlrd>=2.0.1", # legacy .xls +] # OpenTelemetry metrics export otel = [ "opentelemetry-sdk>=1.24.0", @@ -245,10 +250,11 @@ dev = [ "sqlite-vec>=0.1.6", "sentence-transformers>=2.2.0,<6.0", "numpy>=1.24.0", + "openpyxl>=3.1.0", # exercises spreadsheet_ingest (.xlsx) in the test suite ] # All optional dependencies (everything you need) all = [ - "headroom-ai[proxy,code,ml,memory,relevance,image,reports,otel,evals,voice,html,benchmark,mcp]", + "headroom-ai[proxy,code,ml,memory,relevance,image,reports,otel,evals,voice,html,benchmark,mcp,spreadsheet]", ] [project.scripts] diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py new file mode 100644 index 000000000..858708892 --- /dev/null +++ b/tests/test_transforms_tabular.py @@ -0,0 +1,330 @@ +"""Tests for tabular-text + spreadsheet compression. + +Covers detection (content_detector), the CSV→SmartCrusher bridge +(tabular_ingest), router wiring (content_router), and binary spreadsheet +ingestion (spreadsheet_ingest / compress_spreadsheet). +""" + +from __future__ import annotations + +import importlib.util + +import pytest + +from headroom.transforms.content_detector import ( + ContentType, + DetectionResult, + _is_md_separator, + _looks_like_prose, + _try_detect_delimited, + _try_detect_markdown_table, + detect_content_type, +) +from headroom.transforms.content_router import ( + CompressionStrategy, + ContentRouter, + ContentRouterConfig, +) +from headroom.transforms.tabular_ingest import ( + TabularCompressionResult, + TabularCompressor, + parse_csv, + parse_fixed_width, + parse_markdown_table, + parse_tabular, + to_records, +) + +_HAS_OPENPYXL = importlib.util.find_spec("openpyxl") is not None + + +# Reusable fixtures ---------------------------------------------------------- + +CSV = "name,age,city\nAlice,30,NYC\nBob,25,LA\nCara,40,SF" +TSV = "id\tval\tnote\n1\ta\tx\n2\tb\ty\n3\tc\tz" +MARKDOWN = "| name | age |\n| --- | --- |\n| Alice | 30 |\n| Bob | 25 |\n| Cara | 40 |" + + +def _verbose_markdown(rows: int = 40) -> str: + body = "\n".join( + f"| user_{i} | {20 + i} | city_{i % 5} | active | engineering |" for i in range(rows) + ) + return "| name | age | city | status | dept |\n| --- | --- | --- | --- | --- |\n" + body + + +# Detection ------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "content,fmt", + [(CSV, "csv"), (TSV, "csv"), (MARKDOWN, "markdown")], +) +def test_detects_tabular(content: str, fmt: str) -> None: + result = detect_content_type(content) + assert result.content_type is ContentType.TABULAR + assert result.metadata.get("format") == fmt + assert result.confidence >= 0.6 + + +@pytest.mark.parametrize( + "content,expected", + [ + # Search output must not be stolen by tabular. + ( + "src/main.py:42:def process():\nsrc/util.py:10:import os\nsrc/x.py:5:return 1", + ContentType.SEARCH_RESULTS, + ), + # Build/log output stays a log. + ( + "2026-01-01 INFO starting\n2026-01-01 WARN slow\n2026-01-01 ERROR boom", + ContentType.BUILD_OUTPUT, + ), + # JSON arrays still go to the JSON path. + ('[{"a": 1}, {"a": 2}, {"a": 3}]', ContentType.JSON_ARRAY), + # Prose with incidental commas must NOT be tabular. + ( + "Hello there, friend.\nThis is a sentence, yes.\nAnother line, ok.", + ContentType.PLAIN_TEXT, + ), + ], +) +def test_does_not_misroute_to_tabular(content: str, expected: ContentType) -> None: + assert detect_content_type(content).content_type is expected + + +# Detection — edge branches -------------------------------------------------- + + +def test_is_md_separator_needs_two_columns() -> None: + assert _is_md_separator("| --- | --- |") + assert not _is_md_separator("| --- |") # single column is not a separator + assert not _is_md_separator("| a | b |") # cells must be dashes + + +def test_markdown_table_needs_multiple_columns() -> None: + # Valid separator below, but the header is a single column -> not a table. + assert _try_detect_markdown_table(["x|", "---|---", "y|"]) is None + + +def test_delimited_needs_three_rows() -> None: + assert _try_detect_delimited(["a,b,c", "1,2,3"]) is None + + +def test_delimited_rejects_delimiter_only_in_header() -> None: + # Header has commas but the data rows don't: no stable column count. + assert _try_detect_delimited(["a,b,c", "plain", "text"]) is None + + +def test_delimited_rejects_inconsistent_columns() -> None: + # Column count swings too much to be a real table. + assert _try_detect_delimited(["a,b", "c,d", "e,f,g,h", "i,j,k,l,m"]) is None + + +def test_delimited_keeps_first_equal_confidence_delimiter() -> None: + # Comma and semicolon are both consistent; the comma candidate is set first + # and a later, no-better delimiter does not displace it. + result = _try_detect_delimited(["a,b;c", "d,e;f", "g,h;i"]) + assert result is not None + assert result.metadata["delimiter"] == "," + + +def test_looks_like_prose_distinguishes_sentences_from_rows() -> None: + # Wordy cells (avg > 3 words/cell) read as prose even without end punctuation. + assert _looks_like_prose(["the quick brown fox runs, over the lazy dog now"], ",") + # Short field tuples are real CSV rows, not prose. + assert not _looks_like_prose(["a,b,c", "1,2,3", "x,y,z"], ",") + + +# Parsers -------------------------------------------------------------------- + + +def test_parse_csv_and_records() -> None: + headers, rows = parse_csv(CSV) + assert headers == ["name", "age", "city"] + assert rows[0] == ["Alice", "30", "NYC"] + records = to_records(headers, rows) + assert records[1] == {"name": "Bob", "age": "25", "city": "LA"} + + +def test_parse_markdown_table_drops_separator() -> None: + headers, rows = parse_markdown_table(MARKDOWN) + assert headers == ["name", "age"] + assert ["Alice", "30"] in rows + assert all("---" not in cell for row in rows for cell in row) + + +def test_parse_tabular_returns_none_for_non_tabular() -> None: + assert parse_tabular("just a normal paragraph here") is None + + +def test_parse_fixed_width() -> None: + headers, rows = parse_fixed_width("name age city\nAlice 30 NYC\nBob 25 LA") + assert headers == ["name", "age", "city"] + assert rows[0] == ["Alice", "30", "NYC"] + + +def test_to_records_empty_headers_returns_empty() -> None: + assert to_records([], [["a", "b"]]) == [] + + +def test_parse_csv_blank_returns_empty() -> None: + assert parse_csv(" \n \n") == ([], []) + + +def test_parse_markdown_table_too_short_returns_empty() -> None: + assert parse_markdown_table("| only one row |") == ([], []) + + +def test_parse_fixed_width_too_short_returns_empty() -> None: + assert parse_fixed_width("a single line") == ([], []) + + +def test_parse_tabular_dispatches_fixed_width(monkeypatch) -> None: + # The detector currently emits only csv/markdown, so drive the fixed_width + # dispatch branch directly with a stubbed detection result. + import headroom.transforms.tabular_ingest as ti + + monkeypatch.setattr( + ti, + "detect_content_type", + lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "fixed_width"}), + ) + headers, rows, fmt = ti.parse_tabular("name age\nAlice 30\nBob 25") + assert fmt == "fixed_width" + assert headers == ["name", "age"] + assert rows[0] == ["Alice", "30"] + + +def test_parse_tabular_none_when_no_data_rows_survive() -> None: + # Detected as a markdown table, but it is header + separator rows only: + # nothing survives as a data row, so parse_tabular bails to None. + assert parse_tabular("| a | b |\n| --- | --- |\n| --- | --- |") is None + + +def test_compression_ratio_zero_for_empty_original() -> None: + result = TabularCompressionResult( + compressed="", original="", was_modified=False, fmt="csv", rows=0, columns=0 + ) + assert result.compression_ratio == 0.0 + + +# Bridge compressor ---------------------------------------------------------- + + +def test_verbose_markdown_compresses() -> None: + result = TabularCompressor().compress(_verbose_markdown()) + assert result.was_modified + assert len(result.compressed) < len(result.original) + assert result.compression_ratio < 1.0 + assert result.fmt == "markdown" + + +def test_compact_unique_csv_passes_through() -> None: + # All-unique compact rows have nothing losslessly removable. + result = TabularCompressor().compress(CSV) + assert not result.was_modified + assert result.compressed == CSV + + +def test_non_tabular_passes_through_unmodified() -> None: + # Unparseable prose returns the original content untouched. + text = "just a normal paragraph here" + result = TabularCompressor().compress(text) + assert not result.was_modified + assert result.compressed == text + + +# Router wiring -------------------------------------------------------------- + + +def test_router_routes_tabular() -> None: + result = ContentRouter().compress(_verbose_markdown()) + assert result.strategy_used is CompressionStrategy.TABULAR + assert result.total_compressed_tokens <= result.total_original_tokens + + +def test_router_caches_tabular_compressor() -> None: + router = ContentRouter() + first = router._get_tabular_compressor() + assert first is router._get_tabular_compressor() # second call returns the cached instance + + +def test_router_tabular_passthrough_when_compressor_unavailable(monkeypatch) -> None: + # Defensive guard: if the tabular compressor can't be constructed, routing to + # TABULAR leaves content untouched instead of crashing. + md = _verbose_markdown() + router = ContentRouter() + monkeypatch.setattr(router, "_get_tabular_compressor", lambda: None) + result = router.compress(md) + assert result.compressed == md + assert result.tokens_saved == 0 + + +def test_router_respects_disable_flag() -> None: + # Disabling skips the tabular compressor: content passes through unchanged + # (the selected strategy label may still read TABULAR, like other disabled + # compressors). + md = _verbose_markdown() + cfg = ContentRouterConfig(enable_tabular_compressor=False) + result = ContentRouter(cfg).compress(md) + assert result.compressed == md + assert result.tokens_saved == 0 + + +# Binary spreadsheet ingestion ----------------------------------------------- + + +@pytest.mark.skipif(not _HAS_OPENPYXL, reason="openpyxl not installed") +def test_load_and_compress_xlsx(tmp_path) -> None: + import openpyxl + + from headroom import compress_spreadsheet + from headroom.transforms.spreadsheet_ingest import load_spreadsheet + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Data" + ws.append(["id", "name", "dept", "status"]) + for i in range(40): + ws.append([i, f"user_{i}", ["eng", "sales", "ops"][i % 3], "active"]) + wb.create_sheet("Empty") # should be skipped + path = tmp_path / "sample.xlsx" + wb.save(path) + + sheets = load_spreadsheet(path) + assert list(sheets) == ["Data"] + assert sheets["Data"].splitlines()[0] == "id,name,dept,status" + + result = compress_spreadsheet(str(path)) + assert result.tokens_after <= result.tokens_before + + +@pytest.mark.skipif(not _HAS_OPENPYXL, reason="openpyxl not installed") +def test_compress_spreadsheet_empty_workbook_returns_empty(tmp_path) -> None: + import openpyxl + + from headroom import compress_spreadsheet + + wb = openpyxl.Workbook() # one empty sheet, no rows + path = tmp_path / "empty.xlsx" + wb.save(path) + + result = compress_spreadsheet(str(path)) + assert result.messages == [] + assert result.tokens_saved == 0 + + +def test_load_spreadsheet_rejects_unknown_extension(tmp_path) -> None: + from headroom.transforms.spreadsheet_ingest import load_spreadsheet + + bad = tmp_path / "data.txt" + bad.write_text("a,b\n1,2\n") + with pytest.raises(ValueError, match="Unsupported"): + load_spreadsheet(bad) + + +def test_load_spreadsheet_missing_file(tmp_path) -> None: + from headroom.transforms.spreadsheet_ingest import load_spreadsheet + + with pytest.raises(FileNotFoundError): + load_spreadsheet(tmp_path / "nope.xlsx")