From 7dc9a978ca974a2ed264bb585b187dd11e0a04f2 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Fri, 24 Jul 2026 20:40:49 -0700 Subject: [PATCH] feat(lossless): factor shared directory prefix in the grep search fold (#2547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description The lossless search fold (`search_heading`) factors a repeated **file** (many matches in one file → path once + `line:content` rows), but `grep -rn` across many **distinct** files has one match each, so it saved ~0% — the shared directory repeated on every row. This adds `search_dir_heading`/`search_dir_unheading`, which factor the shared **directory** across distinct files (dir once as a header, `base:line:content` beneath). `compact_lossless('search')` now tries both folds and keeps the smallest that round-trips exactly. Matters because grep is ~23.5% of observed agent output tokens. Closes # ## Type of Change - [x] Performance improvement (lossless) ## Changes / Behavior - File fold wins many-matches-one-file; dir fold wins the `grep -rn` case (0% → ~16-40% depending on path depth / match length). **Byte-lossless** — round-trip verified, fold discarded on any mismatch. - Never touches source reads / diffs (unchanged class gating). ## Testing ```text pytest tests/test_bash_search_lossless_fold.py -q → 30 passed pytest test_lossless_excluded_compaction / _then_lossy / _mode → 72 passed ruff + mypy → clean ``` Round-trip verified on: distinct-files (sorted), many-matches-one-file, mixed+passthrough, colon-in-content. ## Note for reviewers The dir-grouped output is byte-lossless but a slightly **non-standard** format the model reads directly (`dir/` header + `base:line:content`) — like the existing `rg --heading` fold but less standard. Low comprehension risk; flagging it explicitly. If preferred, we can gate it to only fire above a larger savings threshold. ## Checklist - [x] Self-reviewed; tests pass; no CHANGELOG edit --- headroom/transforms/lossless_compaction.py | 86 +++++++++++++++++++++- tests/test_bash_search_lossless_fold.py | 42 +++++++++++ 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/headroom/transforms/lossless_compaction.py b/headroom/transforms/lossless_compaction.py index 573f09aee..76c167e15 100644 --- a/headroom/transforms/lossless_compaction.py +++ b/headroom/transforms/lossless_compaction.py @@ -25,6 +25,8 @@ __all__ = [ "unfold_repeated_blocks", "search_heading", "search_unheading", + "search_dir_heading", + "search_dir_unheading", "diff_strip_index", "compact_lossless", ] @@ -283,6 +285,74 @@ def search_unheading(text: str) -> str: return _join(out, had_trailing) +# A dir-heading data row: ``::`` where base has no '/'. +_DIR_DATA_RE = re.compile(r"^(?P[^/\n:]+):(?P\d+):(?P.*)$") + + +def search_dir_heading(text: str) -> str: + """Fold grep ``path:line:content`` rows by DIRECTORY. + + Consecutive rows whose path shares a parent directory collapse to that + directory once (a header ending in ``/``), then ``base:line:content`` rows + beneath it. Complements :func:`search_heading` (which factors a repeated + *file*): this factors a repeated *directory* across distinct files — the + common ``grep -rn`` case where each file has a single match, so file-heading + saves nothing but the shared directory repeats on every row. Rows whose path + has no ``/`` pass through untouched. Exactly reversed by + :func:`search_dir_unheading`; ``compact_lossless`` verifies the round-trip. + """ + lines, had_trailing = _split_keep_trailing(text) + if not lines: + return text + out: list[str] = [] + current_dir: str | None = None + for line in lines: + m = _GREP_ROW_RE.match(line) + if m and "/" in m.group("path"): + path = m.group("path") + cut = path.rindex("/") + 1 + dir_part, base = path[:cut], path[cut:] + if dir_part != current_dir: + out.append(dir_part) + current_dir = dir_part + out.append(f"{base}:{m.group('line')}:{m.group('content')}") + else: + out.append(line) + current_dir = None + return _join(out, had_trailing) + + +def search_dir_unheading(text: str) -> str: + """Exact inverse of :func:`search_dir_heading`. + + A *header* is a line ending in ``/`` immediately followed by a + ``base:line:content`` data row; it is consumed and re-prefixed onto each + following data row until a non-data line appears. + """ + lines, had_trailing = _split_keep_trailing(text) + if not lines: + return text + out: list[str] = [] + current_dir: str | None = None + n = len(lines) + i = 0 + while i < n: + line = lines[i] + data = _DIR_DATA_RE.match(line) + if current_dir is not None and data: + out.append(f"{current_dir}{line}") + i += 1 + continue + if line.endswith("/") and i + 1 < n and _DIR_DATA_RE.match(lines[i + 1]): + current_dir = line + i += 1 + continue + current_dir = None + out.append(line) + i += 1 + return _join(out, had_trailing) + + def diff_strip_index(text: str) -> str: """Drop ``index ..`` lines from a unified diff (still applies).""" lines, had_trailing = _split_keep_trailing(text) @@ -387,10 +457,18 @@ def compact_lossless(content: str, kind: str) -> str: return candidate if _smaller(candidate, content) else content if kind == "search": - candidate = search_heading(content) - if search_unheading(candidate) != content: - return content - return candidate if _smaller(candidate, content) else content + # Two independent folds; keep the smaller that round-trips exactly. + # search_heading factors a repeated FILE (many matches in one file); + # search_dir_heading factors a repeated DIRECTORY (one match each + # across many files in a dir — the grep -rn case the file fold misses). + best = content + for candidate, inverse in ( + (search_heading(content), search_unheading), + (search_dir_heading(content), search_dir_unheading), + ): + if inverse(candidate) == content and _smaller(candidate, best): + best = candidate + return best if kind == "paths": # Pure path listings (find/ls -1/rg -l): fold repeated parent dirs. diff --git a/tests/test_bash_search_lossless_fold.py b/tests/test_bash_search_lossless_fold.py index 0094fa84d..e2e74b361 100644 --- a/tests/test_bash_search_lossless_fold.py +++ b/tests/test_bash_search_lossless_fold.py @@ -221,3 +221,45 @@ def test_experimental_read_keep_ratio_flag_and_gating(monkeypatch): assert r_on._experimental_compress_read("z" * 500) is None # sub-floor content never attempted assert r_on._experimental_compress_read("short") is None + + +# --- directory-prefix fold: grep -rn across many distinct files --- +from headroom.transforms.lossless_compaction import ( # noqa: E402 + compact_lossless, + search_dir_heading, + search_dir_unheading, +) + + +def test_search_dir_fold_factors_directory_across_distinct_files() -> None: + # Sorted grep -rn output: same-dir files are consecutive, one match each, so + # the file-heading fold saves nothing but the shared directory repeats on + # every row. The dir fold factors it out — byte-losslessly. + grep = ( + "\n".join(f"headroom/proxy/mod_{i:02d}.py:{i + 1}: x = compress(p)" for i in range(12)) + + "\n" + ) + folded = compact_lossless(grep, "search") + assert len(folded) < len(grep) # actually shrank (0% before this fold) + assert "headroom/proxy/" in folded # directory factored to a header line + assert search_dir_unheading(folded) == grep # exact byte round-trip + assert search_dir_unheading(search_dir_heading(grep)) == grep + + +def test_search_dir_fold_roundtrips_mixed_and_passthrough() -> None: + mixed = ( + "src/a/x.py:1:hit one\nsrc/a/y.py:2:hit two\n" + "== a plain banner ==\n" + "src/b/z.py:3:content with a colon: value\nnoslash.py:4:pathless row\n" + ) + out = compact_lossless(mixed, "search") + assert search_dir_unheading(out) == mixed or search_unheading(out) == mixed or out == mixed + + +def test_search_file_fold_still_wins_for_many_matches_one_file() -> None: + # Many matches in ONE file: the file fold is smaller, and compact_lossless + # keeps whichever candidate round-trips and is smallest. + grep = "\n".join(f"headroom/proxy/server.py:{i}: line {i}" for i in range(1, 40)) + "\n" + out = compact_lossless(grep, "search") + assert len(out) < len(grep) + assert search_unheading(out) == grep