From 65b0e8c58dbbc0b77e4b7159b279287979767c4c Mon Sep 17 00:00:00 2001 From: Ashish Date: Mon, 15 Jun 2026 08:23:26 -0700 Subject: [PATCH] fix(compression): measure short-value threshold on payload, not token (#889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description `JSONStructureHandler._should_preserve_token` compared `len(token.text)` — which includes both quote characters — against `short_value_threshold`. A value of exactly threshold length was rejected: the documented "20-char threshold" was effectively 18 chars of payload. Stacked on #887. Closes # ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `headroom/compression/handlers/json_handler.py`: strip quotes once at the top of the string-value branch and use the payload length for both the short-value and entropy checks. - `tests/test_compression/test_json_handler.py`: regression test for a value of exactly `short_value_threshold` length. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ pytest tests/test_compression/test_json_handler.py -q 33 passed ``` ## Real Behavior Proof - Environment: local macOS, Python 3.11, branch `fix/json-quote-threshold` (stacked on #887). - Exact command / steps: `pytest tests/test_compression/test_json_handler.py -q`. - Observed result: A 20-char value is preserved at a 20-char threshold; previously it was dropped due to the +2 quote miscount. - Not tested: End-to-end through the live proxy pipeline. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Screenshots (if applicable) N/A — library change. See Test Output. ## Additional Notes Stacked on #887 — review the top commit until that merges. PR 2 of 7. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 --- headroom/compression/handlers/json_handler.py | 9 ++++++--- tests/test_compression/test_json_handler.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/headroom/compression/handlers/json_handler.py b/headroom/compression/handlers/json_handler.py index 67e41af37..80d4f7ec5 100644 --- a/headroom/compression/handlers/json_handler.py +++ b/headroom/compression/handlers/json_handler.py @@ -231,14 +231,17 @@ class JSONStructureHandler(BaseStructureHandler): # In deep array, be more aggressive return False + # Strip quotes once: thresholds apply to the payload, not + # the token. Counting the quote characters made a "20-char + # threshold" effectively 18 chars of value. + value = token.text.strip('"') + # Preserve short values - if self.preserve_short_values and len(token.text) <= self.short_value_threshold: + if self.preserve_short_values and len(value) <= self.short_value_threshold: return True # Preserve high-entropy values (UUIDs, hashes) if self.preserve_high_entropy: - # Strip quotes for entropy calculation - value = token.text.strip('"') # Entropy targets identifiers (UUIDs, hashes, API keys). # Self-normalized entropy also scores English prose >0.85, # so gate on the cheapest identifier signal: no spaces. diff --git a/tests/test_compression/test_json_handler.py b/tests/test_compression/test_json_handler.py index c86b7e20f..404babbf7 100644 --- a/tests/test_compression/test_json_handler.py +++ b/tests/test_compression/test_json_handler.py @@ -222,6 +222,26 @@ class TestJSONStructureHandler: "UUID value should be preserved via entropy" ) + def test_short_value_threshold_excludes_quotes(self): + """The short-value threshold measures the payload, not the token. + + Regression: len(token.text) included both quote characters, so a + value of exactly threshold length was rejected (off-by-2). + """ + handler = JSONStructureHandler( + preserve_short_values=True, + short_value_threshold=20, + preserve_high_entropy=False, + ) + exact = "x" * 20 # exactly at threshold — must be preserved + content = f'{{"key": "{exact}"}}' + result = handler.get_mask(content) + + start = content.index(exact) + assert all(result.mask.mask[i] for i in range(start, start + len(exact))), ( + "value of exactly threshold length should be preserved" + ) + def test_metadata_contains_key_count(self, handler): """Test that metadata includes key count.""" content = '{"a": 1, "b": 2, "c": 3}'