mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
f0670404ce
Builds on the now-merged personas (#1732). Two pieces: ### 1. Lossless compaction for EXCLUDED tool output Excluded tools (Read/Grep/Glob/Write/Edit) stay out of *lossy* compression, but their output is compacted by detected shape: | shape | transform | guarantee | |---|---|---| | grep (SEARCH) | ripgrep --heading fold | **byte-lossless** (`search_unheading` recovers) | | log (BUILD_OUTPUT) | ANSI strip + run-collapse | **byte-lossless** modulo non-semantic ANSI | | json | whitespace-minify | **data-lossless** (`json.loads` equal), NOT byte-exact | Source code + glob path-lists → verbatim. grep gated on `_try_detect_search` (the general/Magika classifier calls grep-over-code SOURCE_CODE and would miss it). Off by default (`compact_excluded_lossless`). ### 2. Enable it in the coding/general personas `compact_excluded_lossless=True` on the coding + general profiles, threaded via `proxy_env` + `proxy_pipeline_kwargs` + a per-request `ContentRouter.apply` override. So `HEADROOM_SAVINGS_PROFILE=coding` auto-folds excluded grep/log/json. ## Why The coding persona was getting ~2.5% on OpenCode because its dominant traffic (Grep/Read) is excluded, and RTK (shell-only, lossy) never sees OpenCode's *native* tools. This recovers those savings losslessly. ## Measured (end-to-end via coding-persona kwargs, real `rg` output) 41,589 → 26,562 chars (**−36%**), `router:excluded:lossless_search`, byte-recoverable. ## Accuracy grep/log = byte-lossless → edit-safe. json = data-lossless (edit-caveat for read-then-edit-JSON, documented). Read of source code → untouched (tested). 47 tests (personas + all three tiers + persona-enablement + end-to-end). ruff + mypy clean. **No personas duplication** — rebased onto main after #1732 landed. Supersedes #1755. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
119 lines
3.9 KiB
Python
119 lines
3.9 KiB
Python
"""Information-preserving compaction for EXCLUDED tool output.
|
|
|
|
Excluded tools (Read/Grep/Glob/Write/Edit) are protected from *lossy*
|
|
compression for accuracy. This feature still compacts them by detected shape,
|
|
using only reversible / data-preserving transforms:
|
|
|
|
* SEARCH (grep) -> ripgrep --heading fold [byte-lossless]
|
|
* LOG -> ANSI strip + run-collapse [byte-lossless modulo ANSI color]
|
|
* JSON -> whitespace-minify [data-lossless; same object, NOT byte-exact]
|
|
|
|
Source code and glob path-lists match nothing -> untouched. Always on
|
|
(information-preserving, so it needs no feature gate) in every path.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from headroom.providers import OpenAIProvider
|
|
from headroom.tokenizer import Tokenizer
|
|
from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
|
|
from headroom.transforms.lossless_compaction import expand_runs, search_unheading, strip_ansi
|
|
|
|
GREP = "".join(
|
|
f"src/module_{f}.py:{ln * 3}:matched occurrence with some real content here\n"
|
|
for f in range(6)
|
|
for ln in range(15)
|
|
)
|
|
LOG = "".join(
|
|
f"\x1b[32m2026-07-03 INFO worker {i % 3} processing job batch\x1b[0m\n" for i in range(40)
|
|
)
|
|
LOG += "".join("2026-07-03 WARN transient retry, backing off\n" for _ in range(25))
|
|
JSON = json.dumps(
|
|
{"users": [{"id": i, "name": f"user{i}", "active": i % 2 == 0} for i in range(40)]},
|
|
indent=2,
|
|
)
|
|
CODE = "def foo(x):\n return x + 1\n\nclass Bar:\n value = 42\n" * 30
|
|
GLOB = "\n".join(f"src/module_{i}.py" for i in range(60)) + "\n"
|
|
|
|
|
|
@pytest.fixture
|
|
def tokenizer():
|
|
provider = OpenAIProvider()
|
|
return Tokenizer(provider.get_token_counter("gpt-4o"), "gpt-4o")
|
|
|
|
|
|
def _compact(content: str):
|
|
router = ContentRouter(ContentRouterConfig())
|
|
return router._lossless_compact_excluded(content)
|
|
|
|
|
|
# --- helper: right transform per shape, right guarantee ---
|
|
|
|
|
|
def test_grep_search_fold_is_byte_lossless():
|
|
out, kind = _compact(GREP)
|
|
assert kind == "search"
|
|
assert len(out) < len(GREP)
|
|
assert search_unheading(out) == GREP # byte-exact
|
|
|
|
|
|
def test_log_compaction_recovers_modulo_ansi():
|
|
out, kind = _compact(LOG)
|
|
assert kind == "log"
|
|
assert len(out) < len(LOG)
|
|
assert expand_runs(out) == strip_ansi(LOG) # recover the lines (ANSI dropped)
|
|
|
|
|
|
def test_json_minify_is_data_lossless():
|
|
out, kind = _compact(JSON)
|
|
assert kind == "json"
|
|
assert len(out) < len(JSON)
|
|
assert json.loads(out) == json.loads(JSON) # same object; NOT byte-exact
|
|
|
|
|
|
def test_source_and_glob_untouched():
|
|
assert _compact(CODE) is None
|
|
assert _compact(GLOB) is None
|
|
|
|
|
|
# --- end-to-end through the router pipeline (excluded tools) ---
|
|
|
|
|
|
def _run(content: str, tool: str, tokenizer):
|
|
router = ContentRouter(ContentRouterConfig())
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [{"id": "c1", "function": {"name": tool, "arguments": "{}"}}],
|
|
},
|
|
{"role": "tool", "tool_call_id": "c1", "content": content},
|
|
]
|
|
result = router.apply(messages, tokenizer, compress_user_messages=True)
|
|
return result.messages[1]["content"], result.transforms_applied
|
|
|
|
|
|
def test_pipeline_folds_grep_and_recovers(tokenizer):
|
|
out, transforms = _run(GREP, "grep", tokenizer)
|
|
assert "router:excluded:lossless_search" in transforms
|
|
assert search_unheading(out) == GREP
|
|
|
|
|
|
def test_pipeline_compacts_log_read(tokenizer):
|
|
out, transforms = _run(LOG, "read", tokenizer)
|
|
assert "router:excluded:lossless_log" in transforms
|
|
assert expand_runs(out) == strip_ansi(LOG)
|
|
|
|
|
|
def test_pipeline_minifies_json_read(tokenizer):
|
|
out, transforms = _run(JSON, "read", tokenizer)
|
|
assert "router:excluded:lossless_json" in transforms
|
|
assert json.loads(out) == json.loads(JSON) # data-lossless (same object)
|
|
|
|
|
|
def test_pipeline_leaves_source_read_untouched(tokenizer):
|
|
out, _ = _run(CODE, "read", tokenizer)
|
|
assert out == CODE
|