From 18e1c3c9badc5169466b7f76ae08e0639f4ba104 Mon Sep 17 00:00:00 2001 From: TenderDeve Date: Mon, 27 Jul 2026 19:14:04 +0530 Subject: [PATCH] fix(compression): report source-line span in CCR compression marker (#2597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description The compression marker read `[N items compressed to M. Retrieve more: hash=...]`, where `items` counts whitespace-split **words**, not lines. So five lines of tool output could show as `[122 items compressed to 27...]`. A reader can't map "items" to lines and can't tell "this line was compressed away" from "this line was never in the output" — absence reads as evidence of absence, which per the report led to a materially wrong conclusion. Closes #2586 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Annotate the marker with the source line count — `[N items compressed to M (from L source lines). Retrieve more: hash=...]` — at both marker sites: `KompressCompressor.compress` / `compress_batch` (`kompress_compressor.py`) and the remote path (`kompress_remote.py`). - The machine-parsed `Retrieve more: hash=` token is left byte-for-byte unchanged, so CCR detection/retrieval is unaffected. Scope note: I intentionally kept the existing `items compressed to` phrasing rather than reword the unit, to avoid churning the marker format that's referenced across ~12 test fixtures and the `config.py` template. This is the minimal honesty fix; happy to go further (e.g. line-unit counts or unifying with the `config.py` template) if you'd prefer — see the issue thread where I asked about wording. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality ### Test Output ```text $ uv run pytest tests/test_compression_units.py tests/test_compression_batches.py \ tests/test_ccr_marker_policy.py tests/test_ccr_tool_injection.py tests/test_session_probes.py -q 92 passed $ uv run pytest tests/test_ccr_marker_policy.py -q 8 passed # incl. new test_source_line_span_marker_is_still_detected $ uv run ruff check headroom/transforms/kompress_compressor.py headroom/transforms/kompress_remote.py tests/test_ccr_marker_policy.py All checks passed! ``` ## Real Behavior Proof - Environment: headroom @ main, Python 3.14, uv - Exact command / steps: added a marker in the new enriched format and ran it through the CCR marker detector. - Observed result: the retrieval hash is still detected from `[122 items compressed to 27 (from 5 source lines). Retrieve more: hash=...]`; existing compression/CCR suites unchanged. - Not tested: mypy not run locally; the full model-backed compress() marker path isn't unit-exercised (needs a real backend), so the new test targets the parser boundary instead. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` ## Additional Notes Wording is adjustable per the issue discussion. The `config.py` marker template (a different code path with `Omitted`/`Expires` fields) is left untouched to keep this focused on the Kompress marker the report hit. --- headroom/transforms/kompress_compressor.py | 16 ++++++++++++++-- headroom/transforms/kompress_remote.py | 7 ++++++- tests/test_ccr_marker_policy.py | 7 +++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py index 0ee2ff2e6..0f5f0c91d 100644 --- a/headroom/transforms/kompress_compressor.py +++ b/headroom/transforms/kompress_compressor.py @@ -1452,8 +1452,14 @@ class KompressCompressor(Transform): cache_key = self._store_in_ccr(ccr_source, compressed, ccr_source_tokens) if cache_key: result.cache_key = cache_key + # Report the source line span so a reader can tell content was + # compressed away rather than absent — "items" counts words, which + # does not map to lines and reads as evidence of absence (#2586). + source_lines = ccr_source.count("\n") + 1 + line_word = "line" if source_lines == 1 else "lines" result.compressed += ( - f"\n[{n_words} items compressed to {compressed_count}." + f"\n[{n_words} items compressed to {compressed_count}" + f" (from {source_lines} source {line_word})." f" Retrieve more: hash={cache_key}]" ) @@ -1809,8 +1815,14 @@ class KompressCompressor(Transform): cache_key = self._store_in_ccr(ccr_source, compressed, ccr_source_tokens) if cache_key: result.cache_key = cache_key + # Report the source line span so a reader can tell content was + # compressed away rather than absent — "items" counts words, which + # does not map to lines and reads as evidence of absence (#2586). + source_lines = ccr_source.count("\n") + 1 + line_word = "line" if source_lines == 1 else "lines" result.compressed += ( - f"\n[{n_words} items compressed to {compressed_count}." + f"\n[{n_words} items compressed to {compressed_count}" + f" (from {source_lines} source {line_word})." f" Retrieve more: hash={cache_key}]" ) diff --git a/headroom/transforms/kompress_remote.py b/headroom/transforms/kompress_remote.py index 92f0810ee..d3a5db1af 100644 --- a/headroom/transforms/kompress_remote.py +++ b/headroom/transforms/kompress_remote.py @@ -129,9 +129,14 @@ class RemoteKompressCompressor: cache_key = store_kompress_in_ccr(content, compressed, result.original_tokens) if cache_key: result.cache_key = cache_key + # Report the source line span so a reader can tell content was + # compressed away rather than absent (#2586). + source_lines = content.count("\n") + 1 + line_word = "line" if source_lines == 1 else "lines" result.compressed += ( f"\n[{result.original_tokens} items compressed to " - f"{result.compressed_tokens}. Retrieve more: hash={cache_key}]" + f"{result.compressed_tokens} (from {source_lines} source {line_word})." + f" Retrieve more: hash={cache_key}]" ) return result diff --git a/tests/test_ccr_marker_policy.py b/tests/test_ccr_marker_policy.py index a9a0e033c..09d89d592 100644 --- a/tests/test_ccr_marker_policy.py +++ b/tests/test_ccr_marker_policy.py @@ -27,6 +27,13 @@ def test_has_new_ccr_markers_filters_replayed_forwarded_markers() -> None: ) +def test_source_line_span_marker_is_still_detected() -> None: + # The compressor annotates the count with a source-line span (#2586); the + # retrieval hash must still be extracted from the enriched marker. + marker = "[122 items compressed to 27 (from 5 source lines). Retrieve more: hash=c00eb437e5e5c00eb437e5e5]" + assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"] + + def test_has_new_ccr_markers_detects_hash_not_seen_in_previous_forward() -> None: old = "[100 items compressed to 10. Retrieve more: hash=abc123def456abc123def456]" new = "[50 items compressed to 5. Retrieve more: hash=deadbeefdeadbeefdeadbeef]"